2024-11-27 21:34:21,745 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-27 21:34:21,765 main DEBUG Took 0.016340 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-27 21:34:21,766 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-27 21:34:21,766 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-27 21:34:21,767 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-27 21:34:21,769 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 21:34:21,778 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-27 21:34:21,793 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 21:34:21,795 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 21:34:21,796 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 21:34:21,796 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 21:34:21,797 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 21:34:21,797 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 21:34:21,799 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 21:34:21,799 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 21:34:21,800 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 21:34:21,800 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 21:34:21,801 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 21:34:21,802 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 21:34:21,803 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 21:34:21,803 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 21:34:21,804 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 21:34:21,804 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 21:34:21,805 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 21:34:21,805 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 21:34:21,806 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 21:34:21,806 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 21:34:21,807 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 21:34:21,807 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 21:34:21,808 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 21:34:21,809 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 21:34:21,810 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 21:34:21,810 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-27 21:34:21,812 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 21:34:21,814 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-27 21:34:21,817 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-27 21:34:21,817 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-27 21:34:21,819 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-27 21:34:21,819 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-27 21:34:21,829 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-27 21:34:21,832 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-27 21:34:21,834 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-27 21:34:21,834 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-27 21:34:21,835 main DEBUG createAppenders(={Console}) 2024-11-27 21:34:21,836 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-27 21:34:21,836 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-27 21:34:21,837 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-27 21:34:21,837 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-27 21:34:21,838 main DEBUG OutputStream closed 2024-11-27 21:34:21,838 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-27 21:34:21,839 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-27 21:34:21,839 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-27 21:34:21,925 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-27 21:34:21,927 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-27 21:34:21,929 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-27 21:34:21,930 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-27 21:34:21,931 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-27 21:34:21,931 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-27 21:34:21,932 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-27 21:34:21,932 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-27 21:34:21,932 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-27 21:34:21,933 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-27 21:34:21,933 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-27 21:34:21,934 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-27 21:34:21,934 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-27 21:34:21,935 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-27 21:34:21,935 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-27 21:34:21,936 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-27 21:34:21,936 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-27 21:34:21,937 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-27 21:34:21,940 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-27 21:34:21,941 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-27 21:34:21,941 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-27 21:34:21,942 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-27T21:34:22,228 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/208c7222-23c4-3618-cb4b-4763d96cc3c1 2024-11-27 21:34:22,232 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-27 21:34:22,232 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-27T21:34:22,244 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-27T21:34:22,275 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-27T21:34:22,297 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/208c7222-23c4-3618-cb4b-4763d96cc3c1/cluster_148e8983-f870-82f4-55b3-08390f1cbfef, deleteOnExit=true 2024-11-27T21:34:22,297 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-27T21:34:22,298 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/208c7222-23c4-3618-cb4b-4763d96cc3c1/test.cache.data in system properties and HBase conf 2024-11-27T21:34:22,299 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/208c7222-23c4-3618-cb4b-4763d96cc3c1/hadoop.tmp.dir in system properties and HBase conf 2024-11-27T21:34:22,300 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/208c7222-23c4-3618-cb4b-4763d96cc3c1/hadoop.log.dir in system properties and HBase conf 2024-11-27T21:34:22,301 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/208c7222-23c4-3618-cb4b-4763d96cc3c1/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-27T21:34:22,302 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/208c7222-23c4-3618-cb4b-4763d96cc3c1/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-27T21:34:22,302 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-27T21:34:22,421 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-27T21:34:22,537 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T21:34:22,542 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/208c7222-23c4-3618-cb4b-4763d96cc3c1/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-27T21:34:22,542 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/208c7222-23c4-3618-cb4b-4763d96cc3c1/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-27T21:34:22,543 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/208c7222-23c4-3618-cb4b-4763d96cc3c1/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-27T21:34:22,544 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/208c7222-23c4-3618-cb4b-4763d96cc3c1/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-27T21:34:22,544 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/208c7222-23c4-3618-cb4b-4763d96cc3c1/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-27T21:34:22,545 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/208c7222-23c4-3618-cb4b-4763d96cc3c1/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-27T21:34:22,546 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/208c7222-23c4-3618-cb4b-4763d96cc3c1/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-27T21:34:22,546 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/208c7222-23c4-3618-cb4b-4763d96cc3c1/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-27T21:34:22,547 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/208c7222-23c4-3618-cb4b-4763d96cc3c1/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-27T21:34:22,547 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/208c7222-23c4-3618-cb4b-4763d96cc3c1/nfs.dump.dir in system properties and HBase conf 2024-11-27T21:34:22,548 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/208c7222-23c4-3618-cb4b-4763d96cc3c1/java.io.tmpdir in system properties and HBase conf 2024-11-27T21:34:22,548 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/208c7222-23c4-3618-cb4b-4763d96cc3c1/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-27T21:34:22,549 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/208c7222-23c4-3618-cb4b-4763d96cc3c1/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-27T21:34:22,550 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/208c7222-23c4-3618-cb4b-4763d96cc3c1/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-27T21:34:23,419 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-27T21:34:23,501 INFO [Time-limited test {}] log.Log(170): Logging initialized @2585ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-27T21:34:23,575 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T21:34:23,642 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T21:34:23,667 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T21:34:23,668 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T21:34:23,669 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-27T21:34:23,686 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T21:34:23,689 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1a58fd8a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/208c7222-23c4-3618-cb4b-4763d96cc3c1/hadoop.log.dir/,AVAILABLE} 2024-11-27T21:34:23,691 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@130d168e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-27T21:34:23,907 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2341a969{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/208c7222-23c4-3618-cb4b-4763d96cc3c1/java.io.tmpdir/jetty-localhost-41757-hadoop-hdfs-3_4_1-tests_jar-_-any-10796251790021352982/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-27T21:34:23,914 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d064751{HTTP/1.1, (http/1.1)}{localhost:41757} 2024-11-27T21:34:23,914 INFO [Time-limited test {}] server.Server(415): Started @2999ms 2024-11-27T21:34:24,317 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T21:34:24,323 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T21:34:24,324 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T21:34:24,324 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T21:34:24,324 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-27T21:34:24,325 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@54a7b62e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/208c7222-23c4-3618-cb4b-4763d96cc3c1/hadoop.log.dir/,AVAILABLE} 2024-11-27T21:34:24,326 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@72782cb0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-27T21:34:24,450 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@292d50c4{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/208c7222-23c4-3618-cb4b-4763d96cc3c1/java.io.tmpdir/jetty-localhost-33789-hadoop-hdfs-3_4_1-tests_jar-_-any-13354151861235639406/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T21:34:24,451 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@667c6eba{HTTP/1.1, (http/1.1)}{localhost:33789} 2024-11-27T21:34:24,451 INFO [Time-limited test {}] server.Server(415): Started @3536ms 2024-11-27T21:34:24,515 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-27T21:34:25,044 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/208c7222-23c4-3618-cb4b-4763d96cc3c1/cluster_148e8983-f870-82f4-55b3-08390f1cbfef/data/data2/current/BP-1289921252-172.17.0.2-1732743263147/current, will proceed with Du for space computation calculation, 2024-11-27T21:34:25,044 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/208c7222-23c4-3618-cb4b-4763d96cc3c1/cluster_148e8983-f870-82f4-55b3-08390f1cbfef/data/data1/current/BP-1289921252-172.17.0.2-1732743263147/current, will proceed with Du for space computation calculation, 2024-11-27T21:34:25,104 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-27T21:34:25,162 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe076fad80c886052 with lease ID 0x737dbdb03634f486: Processing first storage report for DS-4ee3586f-9a01-4c6e-9c9d-ce567249bc55 from datanode DatanodeRegistration(127.0.0.1:36377, datanodeUuid=c9e47ae3-d3c7-423b-ba3f-1c4fb447a714, infoPort=37695, infoSecurePort=0, ipcPort=46827, storageInfo=lv=-57;cid=testClusterID;nsid=2082316648;c=1732743263147) 2024-11-27T21:34:25,164 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe076fad80c886052 with lease ID 0x737dbdb03634f486: from storage DS-4ee3586f-9a01-4c6e-9c9d-ce567249bc55 node DatanodeRegistration(127.0.0.1:36377, datanodeUuid=c9e47ae3-d3c7-423b-ba3f-1c4fb447a714, infoPort=37695, infoSecurePort=0, ipcPort=46827, storageInfo=lv=-57;cid=testClusterID;nsid=2082316648;c=1732743263147), blocks: 0, hasStaleStorage: true, processing time: 3 msecs, invalidatedBlocks: 0 2024-11-27T21:34:25,164 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe076fad80c886052 with lease ID 0x737dbdb03634f486: Processing first storage report for DS-d94a84c7-8a5f-4a0e-ad2c-957834e7efa0 from datanode DatanodeRegistration(127.0.0.1:36377, datanodeUuid=c9e47ae3-d3c7-423b-ba3f-1c4fb447a714, infoPort=37695, infoSecurePort=0, ipcPort=46827, storageInfo=lv=-57;cid=testClusterID;nsid=2082316648;c=1732743263147) 2024-11-27T21:34:25,165 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe076fad80c886052 with lease ID 0x737dbdb03634f486: from storage DS-d94a84c7-8a5f-4a0e-ad2c-957834e7efa0 node DatanodeRegistration(127.0.0.1:36377, datanodeUuid=c9e47ae3-d3c7-423b-ba3f-1c4fb447a714, infoPort=37695, infoSecurePort=0, ipcPort=46827, storageInfo=lv=-57;cid=testClusterID;nsid=2082316648;c=1732743263147), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-27T21:34:25,215 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/208c7222-23c4-3618-cb4b-4763d96cc3c1 2024-11-27T21:34:25,288 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/208c7222-23c4-3618-cb4b-4763d96cc3c1/cluster_148e8983-f870-82f4-55b3-08390f1cbfef/zookeeper_0, clientPort=62402, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/208c7222-23c4-3618-cb4b-4763d96cc3c1/cluster_148e8983-f870-82f4-55b3-08390f1cbfef/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/208c7222-23c4-3618-cb4b-4763d96cc3c1/cluster_148e8983-f870-82f4-55b3-08390f1cbfef/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-27T21:34:25,297 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62402 2024-11-27T21:34:25,312 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T21:34:25,317 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T21:34:25,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741825_1001 (size=7) 2024-11-27T21:34:25,991 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d with version=8 2024-11-27T21:34:25,991 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/hbase-staging 2024-11-27T21:34:26,102 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-27T21:34:26,453 INFO [Time-limited test {}] client.ConnectionUtils(128): master/4585c39bed03:0 server-side Connection retries=45 2024-11-27T21:34:26,464 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T21:34:26,465 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T21:34:26,470 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T21:34:26,470 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T21:34:26,471 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T21:34:26,638 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-27T21:34:26,701 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-27T21:34:26,710 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-27T21:34:26,715 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T21:34:26,745 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 17950 (auto-detected) 2024-11-27T21:34:26,747 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-27T21:34:26,766 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34433 2024-11-27T21:34:26,793 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34433 connecting to ZooKeeper ensemble=127.0.0.1:62402 2024-11-27T21:34:26,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:344330x0, quorum=127.0.0.1:62402, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T21:34:26,838 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34433-0x1015b59e6860000 connected 2024-11-27T21:34:26,889 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T21:34:26,893 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T21:34:26,905 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34433-0x1015b59e6860000, quorum=127.0.0.1:62402, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T21:34:26,910 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d, hbase.cluster.distributed=false 2024-11-27T21:34:26,944 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34433-0x1015b59e6860000, quorum=127.0.0.1:62402, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T21:34:26,950 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34433 2024-11-27T21:34:26,950 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34433 2024-11-27T21:34:26,951 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34433 2024-11-27T21:34:26,956 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34433 2024-11-27T21:34:26,960 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34433 2024-11-27T21:34:27,089 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/4585c39bed03:0 server-side Connection retries=45 2024-11-27T21:34:27,091 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T21:34:27,091 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T21:34:27,092 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T21:34:27,092 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T21:34:27,092 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T21:34:27,095 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-27T21:34:27,098 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T21:34:27,099 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39097 2024-11-27T21:34:27,101 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39097 connecting to ZooKeeper ensemble=127.0.0.1:62402 2024-11-27T21:34:27,102 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T21:34:27,105 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T21:34:27,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:390970x0, quorum=127.0.0.1:62402, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T21:34:27,117 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:390970x0, quorum=127.0.0.1:62402, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T21:34:27,121 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39097-0x1015b59e6860001 connected 2024-11-27T21:34:27,123 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-27T21:34:27,136 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-27T21:34:27,138 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39097-0x1015b59e6860001, quorum=127.0.0.1:62402, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-27T21:34:27,145 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39097-0x1015b59e6860001, quorum=127.0.0.1:62402, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T21:34:27,147 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39097 2024-11-27T21:34:27,148 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39097 2024-11-27T21:34:27,148 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39097 2024-11-27T21:34:27,156 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39097 2024-11-27T21:34:27,157 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39097 2024-11-27T21:34:27,175 DEBUG [M:0;4585c39bed03:34433 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;4585c39bed03:34433 2024-11-27T21:34:27,177 INFO [master/4585c39bed03:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/4585c39bed03,34433,1732743266181 2024-11-27T21:34:27,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39097-0x1015b59e6860001, quorum=127.0.0.1:62402, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T21:34:27,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34433-0x1015b59e6860000, quorum=127.0.0.1:62402, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T21:34:27,187 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34433-0x1015b59e6860000, quorum=127.0.0.1:62402, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/4585c39bed03,34433,1732743266181 2024-11-27T21:34:27,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39097-0x1015b59e6860001, quorum=127.0.0.1:62402, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-27T21:34:27,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34433-0x1015b59e6860000, quorum=127.0.0.1:62402, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T21:34:27,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39097-0x1015b59e6860001, quorum=127.0.0.1:62402, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T21:34:27,210 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34433-0x1015b59e6860000, quorum=127.0.0.1:62402, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-27T21:34:27,211 INFO [master/4585c39bed03:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/4585c39bed03,34433,1732743266181 from backup master directory 2024-11-27T21:34:27,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34433-0x1015b59e6860000, quorum=127.0.0.1:62402, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/4585c39bed03,34433,1732743266181 2024-11-27T21:34:27,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39097-0x1015b59e6860001, quorum=127.0.0.1:62402, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T21:34:27,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34433-0x1015b59e6860000, quorum=127.0.0.1:62402, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T21:34:27,215 WARN [master/4585c39bed03:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T21:34:27,215 INFO [master/4585c39bed03:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=4585c39bed03,34433,1732743266181 2024-11-27T21:34:27,217 INFO [master/4585c39bed03:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-27T21:34:27,219 INFO [master/4585c39bed03:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-27T21:34:27,293 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/hbase.id] with ID: e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2 2024-11-27T21:34:27,293 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/.tmp/hbase.id 2024-11-27T21:34:27,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741826_1002 (size=42) 2024-11-27T21:34:27,308 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/.tmp/hbase.id]:[hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/hbase.id] 2024-11-27T21:34:27,370 INFO [master/4585c39bed03:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T21:34:27,377 INFO [master/4585c39bed03:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-27T21:34:27,398 INFO [master/4585c39bed03:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 19ms. 2024-11-27T21:34:27,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39097-0x1015b59e6860001, quorum=127.0.0.1:62402, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T21:34:27,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34433-0x1015b59e6860000, quorum=127.0.0.1:62402, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T21:34:27,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741827_1003 (size=196) 2024-11-27T21:34:27,437 INFO [master/4585c39bed03:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T21:34:27,440 INFO [master/4585c39bed03:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-27T21:34:27,459 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:27,465 INFO [master/4585c39bed03:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T21:34:27,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741828_1004 (size=1189) 2024-11-27T21:34:27,922 INFO [master/4585c39bed03:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/MasterData/data/master/store 2024-11-27T21:34:27,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741829_1005 (size=34) 2024-11-27T21:34:28,349 INFO [master/4585c39bed03:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-27T21:34:28,351 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T21:34:28,353 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-27T21:34:28,353 INFO [master/4585c39bed03:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T21:34:28,353 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T21:34:28,354 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-27T21:34:28,355 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T21:34:28,355 INFO [master/4585c39bed03:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T21:34:28,356 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732743268353Disabling compacts and flushes for region at 1732743268353Disabling writes for close at 1732743268354 (+1 ms)Writing region close event to WAL at 1732743268355 (+1 ms)Closed at 1732743268355 2024-11-27T21:34:28,358 WARN [master/4585c39bed03:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/MasterData/data/master/store/.initializing 2024-11-27T21:34:28,359 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/MasterData/WALs/4585c39bed03,34433,1732743266181 2024-11-27T21:34:28,367 INFO [master/4585c39bed03:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T21:34:28,386 INFO [master/4585c39bed03:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4585c39bed03%2C34433%2C1732743266181, suffix=, logDir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/MasterData/WALs/4585c39bed03,34433,1732743266181, archiveDir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/MasterData/oldWALs, maxLogs=10 2024-11-27T21:34:28,411 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/MasterData/WALs/4585c39bed03,34433,1732743266181/4585c39bed03%2C34433%2C1732743266181.1732743268391, exclude list is [], retry=0 2024-11-27T21:34:28,436 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36377,DS-4ee3586f-9a01-4c6e-9c9d-ce567249bc55,DISK] 2024-11-27T21:34:28,439 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-27T21:34:28,481 INFO [master/4585c39bed03:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/MasterData/WALs/4585c39bed03,34433,1732743266181/4585c39bed03%2C34433%2C1732743266181.1732743268391 2024-11-27T21:34:28,482 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37695:37695)] 2024-11-27T21:34:28,483 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-27T21:34:28,483 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T21:34:28,488 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T21:34:28,489 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T21:34:28,530 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T21:34:28,557 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-27T21:34:28,561 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:34:28,564 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T21:34:28,564 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T21:34:28,568 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-27T21:34:28,568 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:34:28,569 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:34:28,569 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T21:34:28,572 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-27T21:34:28,572 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:34:28,573 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:34:28,574 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T21:34:28,576 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-27T21:34:28,576 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:34:28,577 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:34:28,578 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T21:34:28,582 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-27T21:34:28,584 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-27T21:34:28,590 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T21:34:28,591 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T21:34:28,595 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T21:34:28,599 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T21:34:28,605 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T21:34:28,606 INFO [master/4585c39bed03:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68053376, jitterRate=0.014074325561523438}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T21:34:28,612 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732743268501Initializing all the Stores at 1732743268503 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732743268504 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732743268504Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732743268505 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732743268505Cleaning up temporary data from old regions at 1732743268591 (+86 ms)Region opened successfully at 1732743268612 (+21 ms) 2024-11-27T21:34:28,613 INFO [master/4585c39bed03:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-27T21:34:28,650 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27857d4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4585c39bed03/172.17.0.2:0 2024-11-27T21:34:28,689 INFO [master/4585c39bed03:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-27T21:34:28,703 INFO [master/4585c39bed03:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-27T21:34:28,703 INFO [master/4585c39bed03:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-27T21:34:28,706 INFO [master/4585c39bed03:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T21:34:28,708 INFO [master/4585c39bed03:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-27T21:34:28,713 INFO [master/4585c39bed03:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-27T21:34:28,713 INFO [master/4585c39bed03:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-27T21:34:28,746 INFO [master/4585c39bed03:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-27T21:34:28,758 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34433-0x1015b59e6860000, quorum=127.0.0.1:62402, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-27T21:34:28,760 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-27T21:34:28,764 INFO [master/4585c39bed03:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-27T21:34:28,766 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34433-0x1015b59e6860000, quorum=127.0.0.1:62402, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-27T21:34:28,767 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-27T21:34:28,769 INFO [master/4585c39bed03:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-27T21:34:28,773 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34433-0x1015b59e6860000, quorum=127.0.0.1:62402, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-27T21:34:28,775 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-27T21:34:28,777 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34433-0x1015b59e6860000, quorum=127.0.0.1:62402, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-27T21:34:28,779 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-27T21:34:28,802 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34433-0x1015b59e6860000, quorum=127.0.0.1:62402, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-27T21:34:28,804 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-27T21:34:28,809 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39097-0x1015b59e6860001, quorum=127.0.0.1:62402, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T21:34:28,809 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34433-0x1015b59e6860000, quorum=127.0.0.1:62402, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T21:34:28,809 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39097-0x1015b59e6860001, quorum=127.0.0.1:62402, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T21:34:28,809 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34433-0x1015b59e6860000, quorum=127.0.0.1:62402, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T21:34:28,812 INFO [master/4585c39bed03:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=4585c39bed03,34433,1732743266181, sessionid=0x1015b59e6860000, setting cluster-up flag (Was=false) 2024-11-27T21:34:28,830 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34433-0x1015b59e6860000, quorum=127.0.0.1:62402, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T21:34:28,830 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39097-0x1015b59e6860001, quorum=127.0.0.1:62402, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T21:34:28,837 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-27T21:34:28,839 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=4585c39bed03,34433,1732743266181 2024-11-27T21:34:28,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34433-0x1015b59e6860000, quorum=127.0.0.1:62402, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T21:34:28,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39097-0x1015b59e6860001, quorum=127.0.0.1:62402, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T21:34:28,850 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-27T21:34:28,853 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=4585c39bed03,34433,1732743266181 2024-11-27T21:34:28,859 INFO [master/4585c39bed03:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-27T21:34:28,866 INFO [RS:0;4585c39bed03:39097 {}] regionserver.HRegionServer(746): ClusterId : e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2 2024-11-27T21:34:28,869 DEBUG [RS:0;4585c39bed03:39097 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-27T21:34:28,874 DEBUG [RS:0;4585c39bed03:39097 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-27T21:34:28,874 DEBUG [RS:0;4585c39bed03:39097 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-27T21:34:28,877 DEBUG [RS:0;4585c39bed03:39097 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-27T21:34:28,878 DEBUG [RS:0;4585c39bed03:39097 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49a0e6a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4585c39bed03/172.17.0.2:0 2024-11-27T21:34:28,892 DEBUG [RS:0;4585c39bed03:39097 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;4585c39bed03:39097 2024-11-27T21:34:28,895 INFO [RS:0;4585c39bed03:39097 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-27T21:34:28,895 INFO [RS:0;4585c39bed03:39097 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-27T21:34:28,895 DEBUG [RS:0;4585c39bed03:39097 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-27T21:34:28,897 INFO [RS:0;4585c39bed03:39097 {}] regionserver.HRegionServer(2659): reportForDuty to master=4585c39bed03,34433,1732743266181 with port=39097, startcode=1732743267048 2024-11-27T21:34:28,910 DEBUG [RS:0;4585c39bed03:39097 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-27T21:34:28,943 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T21:34:28,949 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33843, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-27T21:34:28,954 INFO [master/4585c39bed03:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-27T21:34:28,955 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34433 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:28,961 INFO [master/4585c39bed03:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-27T21:34:28,967 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 4585c39bed03,34433,1732743266181 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-27T21:34:28,974 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/4585c39bed03:0, corePoolSize=5, maxPoolSize=5 2024-11-27T21:34:28,975 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/4585c39bed03:0, corePoolSize=5, maxPoolSize=5 2024-11-27T21:34:28,975 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/4585c39bed03:0, corePoolSize=5, maxPoolSize=5 2024-11-27T21:34:28,975 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/4585c39bed03:0, corePoolSize=5, maxPoolSize=5 2024-11-27T21:34:28,976 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/4585c39bed03:0, corePoolSize=10, maxPoolSize=10 2024-11-27T21:34:28,976 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/4585c39bed03:0, corePoolSize=1, maxPoolSize=1 2024-11-27T21:34:28,976 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/4585c39bed03:0, corePoolSize=2, maxPoolSize=2 2024-11-27T21:34:28,976 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/4585c39bed03:0, corePoolSize=1, maxPoolSize=1 2024-11-27T21:34:28,982 INFO [master/4585c39bed03:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732743298982 2024-11-27T21:34:28,984 DEBUG [RS:0;4585c39bed03:39097 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-27T21:34:28,984 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T21:34:28,984 WARN [RS:0;4585c39bed03:39097 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-27T21:34:28,984 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-27T21:34:28,984 INFO [master/4585c39bed03:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-27T21:34:28,985 INFO [master/4585c39bed03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-27T21:34:28,990 INFO [master/4585c39bed03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-27T21:34:28,990 INFO [master/4585c39bed03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-27T21:34:28,991 INFO [master/4585c39bed03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-27T21:34:28,991 INFO [master/4585c39bed03:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-27T21:34:28,992 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:34:28,992 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-27T21:34:28,992 INFO [master/4585c39bed03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-27T21:34:28,997 INFO [master/4585c39bed03:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-27T21:34:28,999 INFO [master/4585c39bed03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-27T21:34:28,999 INFO [master/4585c39bed03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-27T21:34:29,003 INFO [master/4585c39bed03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-27T21:34:29,003 INFO [master/4585c39bed03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-27T21:34:29,005 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/4585c39bed03:0:becomeActiveMaster-HFileCleaner.large.0-1732743269004,5,FailOnTimeoutGroup] 2024-11-27T21:34:29,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741831_1007 (size=1321) 2024-11-27T21:34:29,006 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/4585c39bed03:0:becomeActiveMaster-HFileCleaner.small.0-1732743269005,5,FailOnTimeoutGroup] 2024-11-27T21:34:29,006 INFO [master/4585c39bed03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-27T21:34:29,006 INFO [master/4585c39bed03:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-27T21:34:29,007 INFO [master/4585c39bed03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-27T21:34:29,008 INFO [master/4585c39bed03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-27T21:34:29,085 INFO [RS:0;4585c39bed03:39097 {}] regionserver.HRegionServer(2659): reportForDuty to master=4585c39bed03,34433,1732743266181 with port=39097, startcode=1732743267048 2024-11-27T21:34:29,088 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34433 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 4585c39bed03,39097,1732743267048 2024-11-27T21:34:29,091 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34433 {}] master.ServerManager(517): Registering regionserver=4585c39bed03,39097,1732743267048 2024-11-27T21:34:29,100 DEBUG [RS:0;4585c39bed03:39097 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d 2024-11-27T21:34:29,101 DEBUG [RS:0;4585c39bed03:39097 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39545 2024-11-27T21:34:29,101 DEBUG [RS:0;4585c39bed03:39097 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-27T21:34:29,107 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34433-0x1015b59e6860000, quorum=127.0.0.1:62402, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T21:34:29,108 DEBUG [RS:0;4585c39bed03:39097 {}] zookeeper.ZKUtil(111): regionserver:39097-0x1015b59e6860001, quorum=127.0.0.1:62402, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/4585c39bed03,39097,1732743267048 2024-11-27T21:34:29,108 WARN [RS:0;4585c39bed03:39097 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T21:34:29,109 INFO [RS:0;4585c39bed03:39097 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T21:34:29,109 DEBUG [RS:0;4585c39bed03:39097 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/WALs/4585c39bed03,39097,1732743267048 2024-11-27T21:34:29,111 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [4585c39bed03,39097,1732743267048] 2024-11-27T21:34:29,135 INFO [RS:0;4585c39bed03:39097 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-27T21:34:29,152 INFO [RS:0;4585c39bed03:39097 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-27T21:34:29,164 INFO [RS:0;4585c39bed03:39097 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-27T21:34:29,165 INFO [RS:0;4585c39bed03:39097 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T21:34:29,166 INFO [RS:0;4585c39bed03:39097 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-27T21:34:29,171 INFO [RS:0;4585c39bed03:39097 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-27T21:34:29,173 INFO [RS:0;4585c39bed03:39097 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-27T21:34:29,173 DEBUG [RS:0;4585c39bed03:39097 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/4585c39bed03:0, corePoolSize=1, maxPoolSize=1 2024-11-27T21:34:29,173 DEBUG [RS:0;4585c39bed03:39097 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/4585c39bed03:0, corePoolSize=1, maxPoolSize=1 2024-11-27T21:34:29,173 DEBUG [RS:0;4585c39bed03:39097 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/4585c39bed03:0, corePoolSize=1, maxPoolSize=1 2024-11-27T21:34:29,174 DEBUG [RS:0;4585c39bed03:39097 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/4585c39bed03:0, corePoolSize=1, maxPoolSize=1 2024-11-27T21:34:29,174 DEBUG [RS:0;4585c39bed03:39097 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/4585c39bed03:0, corePoolSize=1, maxPoolSize=1 2024-11-27T21:34:29,174 DEBUG [RS:0;4585c39bed03:39097 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/4585c39bed03:0, corePoolSize=2, maxPoolSize=2 2024-11-27T21:34:29,174 DEBUG [RS:0;4585c39bed03:39097 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0, corePoolSize=1, maxPoolSize=1 2024-11-27T21:34:29,174 DEBUG [RS:0;4585c39bed03:39097 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/4585c39bed03:0, corePoolSize=1, maxPoolSize=1 2024-11-27T21:34:29,174 DEBUG [RS:0;4585c39bed03:39097 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/4585c39bed03:0, corePoolSize=1, maxPoolSize=1 2024-11-27T21:34:29,175 DEBUG [RS:0;4585c39bed03:39097 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/4585c39bed03:0, corePoolSize=1, maxPoolSize=1 2024-11-27T21:34:29,175 DEBUG [RS:0;4585c39bed03:39097 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/4585c39bed03:0, corePoolSize=1, maxPoolSize=1 2024-11-27T21:34:29,175 DEBUG [RS:0;4585c39bed03:39097 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/4585c39bed03:0, corePoolSize=1, maxPoolSize=1 2024-11-27T21:34:29,175 DEBUG [RS:0;4585c39bed03:39097 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/4585c39bed03:0, corePoolSize=3, maxPoolSize=3 2024-11-27T21:34:29,175 DEBUG [RS:0;4585c39bed03:39097 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0, corePoolSize=3, maxPoolSize=3 2024-11-27T21:34:29,176 INFO [RS:0;4585c39bed03:39097 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T21:34:29,176 INFO [RS:0;4585c39bed03:39097 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T21:34:29,176 INFO [RS:0;4585c39bed03:39097 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T21:34:29,177 INFO [RS:0;4585c39bed03:39097 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-27T21:34:29,177 INFO [RS:0;4585c39bed03:39097 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-27T21:34:29,177 INFO [RS:0;4585c39bed03:39097 {}] hbase.ChoreService(168): Chore ScheduledChore name=4585c39bed03,39097,1732743267048-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T21:34:29,198 INFO [RS:0;4585c39bed03:39097 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-27T21:34:29,200 INFO [RS:0;4585c39bed03:39097 {}] hbase.ChoreService(168): Chore ScheduledChore name=4585c39bed03,39097,1732743267048-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T21:34:29,201 INFO [RS:0;4585c39bed03:39097 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T21:34:29,201 INFO [RS:0;4585c39bed03:39097 {}] regionserver.Replication(171): 4585c39bed03,39097,1732743267048 started 2024-11-27T21:34:29,231 INFO [RS:0;4585c39bed03:39097 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T21:34:29,231 INFO [RS:0;4585c39bed03:39097 {}] regionserver.HRegionServer(1482): Serving as 4585c39bed03,39097,1732743267048, RpcServer on 4585c39bed03/172.17.0.2:39097, sessionid=0x1015b59e6860001 2024-11-27T21:34:29,232 DEBUG [RS:0;4585c39bed03:39097 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-27T21:34:29,232 DEBUG [RS:0;4585c39bed03:39097 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 4585c39bed03,39097,1732743267048 2024-11-27T21:34:29,232 DEBUG [RS:0;4585c39bed03:39097 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4585c39bed03,39097,1732743267048' 2024-11-27T21:34:29,233 DEBUG [RS:0;4585c39bed03:39097 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-27T21:34:29,234 DEBUG [RS:0;4585c39bed03:39097 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-27T21:34:29,235 DEBUG [RS:0;4585c39bed03:39097 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-27T21:34:29,235 DEBUG [RS:0;4585c39bed03:39097 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-27T21:34:29,235 DEBUG [RS:0;4585c39bed03:39097 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 4585c39bed03,39097,1732743267048 2024-11-27T21:34:29,235 DEBUG [RS:0;4585c39bed03:39097 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4585c39bed03,39097,1732743267048' 2024-11-27T21:34:29,235 DEBUG [RS:0;4585c39bed03:39097 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-27T21:34:29,236 DEBUG [RS:0;4585c39bed03:39097 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-27T21:34:29,237 DEBUG [RS:0;4585c39bed03:39097 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-27T21:34:29,237 INFO [RS:0;4585c39bed03:39097 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-27T21:34:29,237 INFO [RS:0;4585c39bed03:39097 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-27T21:34:29,345 INFO [RS:0;4585c39bed03:39097 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T21:34:29,349 INFO [RS:0;4585c39bed03:39097 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4585c39bed03%2C39097%2C1732743267048, suffix=, logDir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/WALs/4585c39bed03,39097,1732743267048, archiveDir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/oldWALs, maxLogs=32 2024-11-27T21:34:29,375 DEBUG [RS:0;4585c39bed03:39097 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/WALs/4585c39bed03,39097,1732743267048/4585c39bed03%2C39097%2C1732743267048.1732743269353, exclude list is [], retry=0 2024-11-27T21:34:29,380 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36377,DS-4ee3586f-9a01-4c6e-9c9d-ce567249bc55,DISK] 2024-11-27T21:34:29,386 INFO [RS:0;4585c39bed03:39097 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/WALs/4585c39bed03,39097,1732743267048/4585c39bed03%2C39097%2C1732743267048.1732743269353 2024-11-27T21:34:29,386 DEBUG [RS:0;4585c39bed03:39097 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37695:37695)] 2024-11-27T21:34:29,410 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-27T21:34:29,410 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d 2024-11-27T21:34:29,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741833_1009 (size=32) 2024-11-27T21:34:29,828 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T21:34:29,831 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-27T21:34:29,834 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-27T21:34:29,834 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:34:29,835 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T21:34:29,835 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-27T21:34:29,838 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-27T21:34:29,839 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:34:29,840 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T21:34:29,840 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-27T21:34:29,844 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-27T21:34:29,844 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:34:29,845 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T21:34:29,846 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-27T21:34:29,850 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-27T21:34:29,850 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:34:29,851 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T21:34:29,852 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-27T21:34:29,854 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/hbase/meta/1588230740 2024-11-27T21:34:29,855 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/hbase/meta/1588230740 2024-11-27T21:34:29,858 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-27T21:34:29,859 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-27T21:34:29,860 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T21:34:29,863 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-27T21:34:29,868 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T21:34:29,869 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66388147, jitterRate=-0.010739520192146301}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T21:34:29,872 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732743269828Initializing all the Stores at 1732743269830 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732743269830Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732743269830Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732743269830Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732743269830Cleaning up temporary data from old regions at 1732743269859 (+29 ms)Region opened successfully at 1732743269872 (+13 ms) 2024-11-27T21:34:29,873 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-27T21:34:29,873 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-27T21:34:29,873 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-27T21:34:29,873 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-27T21:34:29,873 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-27T21:34:29,875 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-27T21:34:29,875 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732743269872Disabling compacts and flushes for region at 1732743269872Disabling writes for close at 1732743269873 (+1 ms)Writing region close event to WAL at 1732743269874 (+1 ms)Closed at 1732743269875 (+1 ms) 2024-11-27T21:34:29,879 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T21:34:29,879 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-27T21:34:29,885 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-27T21:34:29,897 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T21:34:29,900 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-27T21:34:30,053 DEBUG [4585c39bed03:34433 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-27T21:34:30,067 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=4585c39bed03,39097,1732743267048 2024-11-27T21:34:30,075 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4585c39bed03,39097,1732743267048, state=OPENING 2024-11-27T21:34:30,083 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-27T21:34:30,085 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39097-0x1015b59e6860001, quorum=127.0.0.1:62402, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T21:34:30,085 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34433-0x1015b59e6860000, quorum=127.0.0.1:62402, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T21:34:30,086 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T21:34:30,086 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T21:34:30,088 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T21:34:30,091 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=4585c39bed03,39097,1732743267048}] 2024-11-27T21:34:30,268 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-27T21:34:30,272 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47055, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-27T21:34:30,282 INFO [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-27T21:34:30,282 INFO [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T21:34:30,283 INFO [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-27T21:34:30,286 INFO [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4585c39bed03%2C39097%2C1732743267048.meta, suffix=.meta, logDir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/WALs/4585c39bed03,39097,1732743267048, archiveDir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/oldWALs, maxLogs=32 2024-11-27T21:34:30,302 DEBUG [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/WALs/4585c39bed03,39097,1732743267048/4585c39bed03%2C39097%2C1732743267048.meta.1732743270288.meta, exclude list is [], retry=0 2024-11-27T21:34:30,306 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36377,DS-4ee3586f-9a01-4c6e-9c9d-ce567249bc55,DISK] 2024-11-27T21:34:30,309 INFO [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/WALs/4585c39bed03,39097,1732743267048/4585c39bed03%2C39097%2C1732743267048.meta.1732743270288.meta 2024-11-27T21:34:30,309 DEBUG [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37695:37695)] 2024-11-27T21:34:30,309 DEBUG [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-27T21:34:30,311 DEBUG [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-27T21:34:30,314 DEBUG [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-27T21:34:30,318 INFO [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-27T21:34:30,323 DEBUG [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-27T21:34:30,323 DEBUG [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T21:34:30,323 DEBUG [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-27T21:34:30,323 DEBUG [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-27T21:34:30,327 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-27T21:34:30,328 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-27T21:34:30,328 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:34:30,329 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T21:34:30,329 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-27T21:34:30,331 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-27T21:34:30,331 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:34:30,332 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T21:34:30,332 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-27T21:34:30,333 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-27T21:34:30,333 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:34:30,334 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T21:34:30,334 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-27T21:34:30,336 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-27T21:34:30,336 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:34:30,337 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T21:34:30,337 DEBUG [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-27T21:34:30,338 DEBUG [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/hbase/meta/1588230740 2024-11-27T21:34:30,341 DEBUG [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/hbase/meta/1588230740 2024-11-27T21:34:30,343 DEBUG [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-27T21:34:30,343 DEBUG [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-27T21:34:30,344 DEBUG [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T21:34:30,347 DEBUG [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-27T21:34:30,349 INFO [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61817264, jitterRate=-0.07885098457336426}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T21:34:30,349 DEBUG [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-27T21:34:30,350 DEBUG [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732743270324Writing region info on filesystem at 1732743270324Initializing all the Stores at 1732743270326 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732743270326Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732743270326Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732743270327 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732743270327Cleaning up temporary data from old regions at 1732743270343 (+16 ms)Running coprocessor post-open hooks at 1732743270349 (+6 ms)Region opened successfully at 1732743270350 (+1 ms) 2024-11-27T21:34:30,358 INFO [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732743270259 2024-11-27T21:34:30,369 DEBUG [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-27T21:34:30,369 INFO [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-27T21:34:30,371 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=4585c39bed03,39097,1732743267048 2024-11-27T21:34:30,373 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4585c39bed03,39097,1732743267048, state=OPEN 2024-11-27T21:34:30,378 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39097-0x1015b59e6860001, quorum=127.0.0.1:62402, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T21:34:30,378 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34433-0x1015b59e6860000, quorum=127.0.0.1:62402, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T21:34:30,379 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T21:34:30,379 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T21:34:30,379 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:30,383 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-27T21:34:30,383 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4585c39bed03,39097,1732743267048 in 289 msec 2024-11-27T21:34:30,389 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-27T21:34:30,390 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 500 msec 2024-11-27T21:34:30,391 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T21:34:30,391 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-27T21:34:30,412 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:34:30,413 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:34:30,434 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:34:30,437 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53959, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:34:30,467 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5730 sec 2024-11-27T21:34:30,467 INFO [master/4585c39bed03:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732743270467, completionTime=-1 2024-11-27T21:34:30,471 INFO [master/4585c39bed03:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-27T21:34:30,471 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T21:34:30,502 INFO [master/4585c39bed03:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-27T21:34:30,502 INFO [master/4585c39bed03:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732743330502 2024-11-27T21:34:30,502 INFO [master/4585c39bed03:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732743390502 2024-11-27T21:34:30,502 INFO [master/4585c39bed03:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 31 msec 2024-11-27T21:34:30,530 INFO [master/4585c39bed03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4585c39bed03,34433,1732743266181-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T21:34:30,531 INFO [master/4585c39bed03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4585c39bed03,34433,1732743266181-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T21:34:30,531 INFO [master/4585c39bed03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4585c39bed03,34433,1732743266181-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T21:34:30,533 INFO [master/4585c39bed03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-4585c39bed03:34433, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T21:34:30,533 INFO [master/4585c39bed03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-27T21:34:30,534 INFO [master/4585c39bed03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-27T21:34:30,541 DEBUG [master/4585c39bed03:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-27T21:34:30,569 INFO [master/4585c39bed03:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.354sec 2024-11-27T21:34:30,570 INFO [master/4585c39bed03:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-27T21:34:30,572 INFO [master/4585c39bed03:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-27T21:34:30,573 INFO [master/4585c39bed03:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-27T21:34:30,573 INFO [master/4585c39bed03:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-27T21:34:30,573 INFO [master/4585c39bed03:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-27T21:34:30,574 INFO [master/4585c39bed03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4585c39bed03,34433,1732743266181-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T21:34:30,575 INFO [master/4585c39bed03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4585c39bed03,34433,1732743266181-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-27T21:34:30,584 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-27T21:34:30,585 INFO [master/4585c39bed03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-27T21:34:30,586 INFO [master/4585c39bed03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4585c39bed03,34433,1732743266181-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T21:34:30,677 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d57a2b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:34:30,680 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-27T21:34:30,680 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-27T21:34:30,685 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:34:30,689 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:34:30,696 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:34:30,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:34:30,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:34:30,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5314b86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:34:30,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:34:30,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:34:30,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:34:30,705 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:34:30,708 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a7fbb0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:34:30,709 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:34:30,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:34:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:34:30,719 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35934, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:34:30,748 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=4585c39bed03,34433,1732743266181 2024-11-27T21:34:30,762 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=265, ProcessCount=11, AvailableMemoryMB=9233 2024-11-27T21:34:30,771 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T21:34:30,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 4585c39bed03,34433,1732743266181 2024-11-27T21:34:30,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2e3afd56 2024-11-27T21:34:30,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T21:34:30,783 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56824, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T21:34:30,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T21:34:30,790 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T21:34:30,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-27T21:34:30,803 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T21:34:30,805 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-27T21:34:30,805 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:34:30,807 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T21:34:30,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T21:34:30,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741835_1011 (size=959) 2024-11-27T21:34:30,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T21:34:31,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T21:34:31,226 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d 2024-11-27T21:34:31,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741836_1012 (size=53) 2024-11-27T21:34:31,243 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T21:34:31,244 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ebbbf218e8d25371b122092896fed576, disabling compactions & flushes 2024-11-27T21:34:31,244 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:31,244 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:31,244 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. after waiting 0 ms 2024-11-27T21:34:31,244 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:31,244 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:31,244 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ebbbf218e8d25371b122092896fed576: Waiting for close lock at 1732743271243Disabling compacts and flushes for region at 1732743271243Disabling writes for close at 1732743271244 (+1 ms)Writing region close event to WAL at 1732743271244Closed at 1732743271244 2024-11-27T21:34:31,247 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T21:34:31,254 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732743271247"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732743271247"}]},"ts":"1732743271247"} 2024-11-27T21:34:31,262 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T21:34:31,265 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T21:34:31,268 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732743271265"}]},"ts":"1732743271265"} 2024-11-27T21:34:31,273 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-27T21:34:31,274 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ebbbf218e8d25371b122092896fed576, ASSIGN}] 2024-11-27T21:34:31,277 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ebbbf218e8d25371b122092896fed576, ASSIGN 2024-11-27T21:34:31,279 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ebbbf218e8d25371b122092896fed576, ASSIGN; state=OFFLINE, location=4585c39bed03,39097,1732743267048; forceNewPlan=false, retain=false 2024-11-27T21:34:31,431 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ebbbf218e8d25371b122092896fed576, regionState=OPENING, regionLocation=4585c39bed03,39097,1732743267048 2024-11-27T21:34:31,436 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ebbbf218e8d25371b122092896fed576, ASSIGN because future has completed 2024-11-27T21:34:31,437 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048}] 2024-11-27T21:34:31,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T21:34:31,599 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:31,600 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} 2024-11-27T21:34:31,600 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ebbbf218e8d25371b122092896fed576 2024-11-27T21:34:31,601 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T21:34:31,601 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for ebbbf218e8d25371b122092896fed576 2024-11-27T21:34:31,601 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for ebbbf218e8d25371b122092896fed576 2024-11-27T21:34:31,603 INFO [StoreOpener-ebbbf218e8d25371b122092896fed576-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ebbbf218e8d25371b122092896fed576 2024-11-27T21:34:31,606 INFO [StoreOpener-ebbbf218e8d25371b122092896fed576-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ebbbf218e8d25371b122092896fed576 columnFamilyName A 2024-11-27T21:34:31,606 DEBUG [StoreOpener-ebbbf218e8d25371b122092896fed576-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:34:31,608 INFO [StoreOpener-ebbbf218e8d25371b122092896fed576-1 {}] regionserver.HStore(327): Store=ebbbf218e8d25371b122092896fed576/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:34:31,608 INFO [StoreOpener-ebbbf218e8d25371b122092896fed576-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ebbbf218e8d25371b122092896fed576 2024-11-27T21:34:31,610 INFO [StoreOpener-ebbbf218e8d25371b122092896fed576-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ebbbf218e8d25371b122092896fed576 columnFamilyName B 2024-11-27T21:34:31,610 DEBUG [StoreOpener-ebbbf218e8d25371b122092896fed576-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:34:31,611 INFO [StoreOpener-ebbbf218e8d25371b122092896fed576-1 {}] regionserver.HStore(327): Store=ebbbf218e8d25371b122092896fed576/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:34:31,611 INFO [StoreOpener-ebbbf218e8d25371b122092896fed576-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ebbbf218e8d25371b122092896fed576 2024-11-27T21:34:31,613 INFO [StoreOpener-ebbbf218e8d25371b122092896fed576-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ebbbf218e8d25371b122092896fed576 columnFamilyName C 2024-11-27T21:34:31,614 DEBUG [StoreOpener-ebbbf218e8d25371b122092896fed576-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:34:31,614 INFO [StoreOpener-ebbbf218e8d25371b122092896fed576-1 {}] regionserver.HStore(327): Store=ebbbf218e8d25371b122092896fed576/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:34:31,615 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for ebbbf218e8d25371b122092896fed576 2024-11-27T21:34:31,616 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576 2024-11-27T21:34:31,616 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576 2024-11-27T21:34:31,618 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for ebbbf218e8d25371b122092896fed576 2024-11-27T21:34:31,618 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for ebbbf218e8d25371b122092896fed576 2024-11-27T21:34:31,619 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T21:34:31,621 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for ebbbf218e8d25371b122092896fed576 2024-11-27T21:34:31,625 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T21:34:31,625 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened ebbbf218e8d25371b122092896fed576; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64196771, jitterRate=-0.0433935672044754}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T21:34:31,626 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ebbbf218e8d25371b122092896fed576 2024-11-27T21:34:31,626 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for ebbbf218e8d25371b122092896fed576: Running coprocessor pre-open hook at 1732743271601Writing region info on filesystem at 1732743271601Initializing all the Stores at 1732743271603 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732743271603Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732743271603Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732743271603Cleaning up temporary data from old regions at 1732743271618 (+15 ms)Running coprocessor post-open hooks at 1732743271626 (+8 ms)Region opened successfully at 1732743271626 2024-11-27T21:34:31,628 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., pid=6, masterSystemTime=1732743271592 2024-11-27T21:34:31,632 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:31,632 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:31,633 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ebbbf218e8d25371b122092896fed576, regionState=OPEN, openSeqNum=2, regionLocation=4585c39bed03,39097,1732743267048 2024-11-27T21:34:31,637 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 because future has completed 2024-11-27T21:34:31,642 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-27T21:34:31,642 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 in 201 msec 2024-11-27T21:34:31,646 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-27T21:34:31,646 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ebbbf218e8d25371b122092896fed576, ASSIGN in 368 msec 2024-11-27T21:34:31,647 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T21:34:31,648 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732743271647"}]},"ts":"1732743271647"} 2024-11-27T21:34:31,651 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-27T21:34:31,652 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T21:34:31,655 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 858 msec 2024-11-27T21:34:31,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T21:34:31,955 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:34:31,961 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1487a66f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:34:31,961 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:34:31,961 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:34:31,962 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:34:31,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:34:31,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:34:31,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51268a5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:34:31,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:34:31,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:34:31,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:34:31,966 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56844, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:34:31,967 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ff1be82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:34:31,967 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:34:31,969 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:34:31,969 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:34:31,971 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35948, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:34:31,973 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T21:34:31,974 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 4585c39bed03,34433,1732743266181 2024-11-27T21:34:31,974 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@254d7cb 2024-11-27T21:34:31,977 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T21:34:31,978 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56848, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T21:34:31,982 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50d3f2f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:34:31,983 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:34:31,983 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:34:31,984 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:34:31,984 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:34:31,984 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:34:31,985 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@654d6699, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:34:31,985 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:34:31,985 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:34:31,985 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:34:31,986 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56872, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:34:31,987 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@688e5843, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:34:31,988 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46ad9e2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:34:31,988 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:34:31,988 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:34:31,990 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:34:31,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:34:31,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:34:31,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39f7011d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:34:31,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:34:31,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:34:31,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:34:31,993 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56888, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:34:31,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6488fd9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:34:31,994 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6308d33d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:34:31,994 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:34:31,994 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:34:31,995 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:34:31,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:34:31,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:34:31,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73fc0926, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:34:31,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:34:31,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:34:31,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:34:31,998 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56894, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:34:31,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4474c69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:34:32,000 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a822d7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:34:32,000 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:34:32,000 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:34:32,002 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:34:32,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:34:32,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:34:32,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48ffac79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:34:32,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:34:32,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:34:32,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:34:32,005 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:34:32,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5580124c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:34:32,008 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7813eeba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:34:32,008 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:34:32,009 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:34:32,010 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:34:32,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:34:32,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:34:32,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68aa74df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:34:32,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:34:32,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:34:32,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:34:32,012 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56918, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:34:32,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@404bac42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:34:32,014 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f66d26b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:34:32,014 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:34:32,015 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:34:32,015 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:34:32,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:34:32,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:34:32,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@230a9a94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:34:32,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:34:32,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:34:32,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:34:32,018 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56944, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:34:32,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3eee2234, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:34:32,019 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@225d1960, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:34:32,020 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:34:32,020 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:34:32,020 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:34:32,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:34:32,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:34:32,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@585eb20a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:34:32,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:34:32,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:34:32,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:34:32,023 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56962, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:34:32,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b16cd13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:34:32,025 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@93b4e10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:34:32,025 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:34:32,026 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:34:32,027 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:34:32,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:34:32,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:34:32,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cbc32ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:34:32,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:34:32,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:34:32,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:34:32,029 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56976, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:34:32,030 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f894d06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:34:32,031 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7449cb3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:34:32,031 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:34:32,031 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:34:32,032 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:34:32,032 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:34:32,032 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:34:32,032 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31d94f17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:34:32,033 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:34:32,033 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:34:32,033 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:34:32,034 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56994, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:34:32,035 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d7c416b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:34:32,039 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:34:32,041 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:34:32,041 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:34:32,042 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:34:32,042 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:34:32,042 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:34:32,043 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35964, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:34:32,044 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:34:32,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:34:32,044 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:34:32,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:34:32,047 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35974, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:34:32,047 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35980, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:34:32,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-27T21:34:32,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2] 2024-11-27T21:34:32,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2] 2024-11-27T21:34:32,051 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:34:32,052 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:34:32,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2] 2024-11-27T21:34:32,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:34:32,054 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:34:32,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:34:32,055 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:34:32,056 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35994, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:34:32,056 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:34:32,056 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:34:32,057 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:34:32,058 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:34:32,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T21:34:32,059 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36000, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:34:32,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2] 2024-11-27T21:34:32,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:34:32,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:34:32,062 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:34:32,062 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:34:32,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:34:32,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:34:32,066 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36030, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:34:32,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2] 2024-11-27T21:34:32,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2] 2024-11-27T21:34:32,074 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:34:32,075 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:34:32,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2] 2024-11-27T21:34:32,077 DEBUG [Thread-147 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:34:32,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:34:32,077 DEBUG [Thread-147 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:34:32,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:34:32,079 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36036, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:34:32,080 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36052, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:34:32,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2] 2024-11-27T21:34:32,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2] 2024-11-27T21:34:32,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T21:34:32,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on ebbbf218e8d25371b122092896fed576 2024-11-27T21:34:32,168 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:34:32,172 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ebbbf218e8d25371b122092896fed576 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T21:34:32,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T21:34:32,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:32,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:32,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:32,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:32,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:32,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:32,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/c3077b7e16ee482baf5e1d6b0b78321c is 50, key is test_row_0/A:col10/1732743272166/Put/seqid=0 2024-11-27T21:34:32,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741837_1013 (size=14341) 2024-11-27T21:34:32,370 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/c3077b7e16ee482baf5e1d6b0b78321c 2024-11-27T21:34:32,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T21:34:32,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:32,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743332371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:32,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:32,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743332393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:32,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:32,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:32,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:32,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:32,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:32,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743332398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:32,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:32,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:32,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:32,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:32,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:32,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:32,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743332399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:32,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:32,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:32,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:32,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:32,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743332388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:32,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:32,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:32,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:32,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T21:34:32,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:32,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:32,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:32,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:32,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:32,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:32,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:32,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743332528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:32,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:32,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:32,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:32,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:32,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743332529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:32,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:32,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:32,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:32,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:32,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743332530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:32,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:32,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:32,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:32,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:32,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/7bf3630f62bd49e6aa1d034d1c399c7c is 50, key is test_row_0/B:col10/1732743272166/Put/seqid=0 2024-11-27T21:34:32,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743332531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:32,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:32,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:32,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:32,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:32,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743332531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:32,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:32,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:32,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:32,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741838_1014 (size=12001) 2024-11-27T21:34:32,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/7bf3630f62bd49e6aa1d034d1c399c7c 2024-11-27T21:34:32,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T21:34:32,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:32,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:32,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:32,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:32,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:32,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:32,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/b3a2393c3b6f41f6a902cf18a68a0d44 is 50, key is test_row_0/C:col10/1732743272166/Put/seqid=0 2024-11-27T21:34:32,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741839_1015 (size=12001) 2024-11-27T21:34:32,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/b3a2393c3b6f41f6a902cf18a68a0d44 2024-11-27T21:34:32,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T21:34:32,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/c3077b7e16ee482baf5e1d6b0b78321c as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/c3077b7e16ee482baf5e1d6b0b78321c 2024-11-27T21:34:32,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/c3077b7e16ee482baf5e1d6b0b78321c, entries=200, sequenceid=14, filesize=14.0 K 2024-11-27T21:34:32,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/7bf3630f62bd49e6aa1d034d1c399c7c as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/7bf3630f62bd49e6aa1d034d1c399c7c 2024-11-27T21:34:32,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/7bf3630f62bd49e6aa1d034d1c399c7c, entries=150, sequenceid=14, filesize=11.7 K 2024-11-27T21:34:32,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/b3a2393c3b6f41f6a902cf18a68a0d44 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/b3a2393c3b6f41f6a902cf18a68a0d44 2024-11-27T21:34:32,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:32,748 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/b3a2393c3b6f41f6a902cf18a68a0d44, entries=150, sequenceid=14, filesize=11.7 K 2024-11-27T21:34:32,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743332735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:32,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:32,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:32,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:32,764 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for ebbbf218e8d25371b122092896fed576 in 590ms, sequenceid=14, compaction requested=false 2024-11-27T21:34:32,766 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-27T21:34:32,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:32,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on ebbbf218e8d25371b122092896fed576 2024-11-27T21:34:32,782 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:34:32,782 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ebbbf218e8d25371b122092896fed576 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T21:34:32,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T21:34:32,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:32,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:32,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:32,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:32,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:32,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:32,813 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/612917100cf9424a8c9e224ff7d205a6 is 50, key is test_row_0/A:col10/1732743272778/Put/seqid=0 2024-11-27T21:34:32,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:32,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743332833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:32,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:32,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:32,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:32,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:32,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743332841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:32,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:32,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:32,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:32,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:32,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743332841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:32,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:32,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:32,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:32,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:32,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743332843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:32,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:32,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:32,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:32,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741840_1016 (size=12001) 2024-11-27T21:34:32,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/612917100cf9424a8c9e224ff7d205a6 2024-11-27T21:34:32,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/fdf1aa591d674cf7a40fbc37e1fd5ce9 is 50, key is test_row_0/B:col10/1732743272778/Put/seqid=0 2024-11-27T21:34:32,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741841_1017 (size=12001) 2024-11-27T21:34:32,922 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/fdf1aa591d674cf7a40fbc37e1fd5ce9 2024-11-27T21:34:32,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T21:34:32,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:32,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:32,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:32,945 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:32,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:32,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/c47faff8a40741549bb282403c543b15 is 50, key is test_row_0/C:col10/1732743272778/Put/seqid=0 2024-11-27T21:34:32,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:32,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:32,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743332956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:32,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:32,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743332956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:32,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:32,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:32,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:32,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:32,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:32,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:32,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:32,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743332957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:32,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:32,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:32,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:32,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:32,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743332957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:32,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:32,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:32,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:32,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741842_1018 (size=12001) 2024-11-27T21:34:33,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:33,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743333064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:33,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:33,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:33,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:33,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T21:34:33,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:33,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:33,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:33,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:33,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:33,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:33,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:33,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743333174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:33,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:33,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:33,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:33,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743333175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:33,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:33,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:33,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:33,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:33,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:33,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743333176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:33,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:33,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:33,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:33,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:33,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743333177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:33,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:33,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:33,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:33,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T21:34:33,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T21:34:33,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:33,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:33,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:33,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:33,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:33,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:33,393 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/c47faff8a40741549bb282403c543b15 2024-11-27T21:34:33,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/612917100cf9424a8c9e224ff7d205a6 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/612917100cf9424a8c9e224ff7d205a6 2024-11-27T21:34:33,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T21:34:33,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:33,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:33,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:33,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:33,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:33,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:33,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/612917100cf9424a8c9e224ff7d205a6, entries=150, sequenceid=39, filesize=11.7 K 2024-11-27T21:34:33,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/fdf1aa591d674cf7a40fbc37e1fd5ce9 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/fdf1aa591d674cf7a40fbc37e1fd5ce9 2024-11-27T21:34:33,463 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/fdf1aa591d674cf7a40fbc37e1fd5ce9, entries=150, sequenceid=39, filesize=11.7 K 2024-11-27T21:34:33,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/c47faff8a40741549bb282403c543b15 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/c47faff8a40741549bb282403c543b15 2024-11-27T21:34:33,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/c47faff8a40741549bb282403c543b15, entries=150, sequenceid=39, filesize=11.7 K 2024-11-27T21:34:33,498 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for ebbbf218e8d25371b122092896fed576 in 715ms, sequenceid=39, compaction requested=false 2024-11-27T21:34:33,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:33,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on ebbbf218e8d25371b122092896fed576 2024-11-27T21:34:33,500 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:34:33,501 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ebbbf218e8d25371b122092896fed576 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-27T21:34:33,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/c9d1d7a712e34571b41d75a7bbecbd63 is 50, key is test_row_0/A:col10/1732743272795/Put/seqid=0 2024-11-27T21:34:33,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741843_1019 (size=16681) 2024-11-27T21:34:33,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T21:34:33,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:33,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:33,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:33,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:33,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:33,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:33,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:33,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743333594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:33,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:33,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:33,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:33,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:33,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743333595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:33,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:33,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:33,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:33,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:33,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743333599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:33,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:33,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:33,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:33,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:33,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743333611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:33,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:33,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:33,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:33,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:33,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743333612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:33,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:33,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:33,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:33,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T21:34:33,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:33,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:33,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:33,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:33,730 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:33,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:33,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743333725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:33,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:33,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743333726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:33,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:33,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:33,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:33,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:33,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:33,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:33,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:33,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:33,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743333727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:33,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:33,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:33,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:33,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:33,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743333744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:33,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:33,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:33,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:33,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:33,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743333745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:33,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:33,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:33,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:33,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T21:34:33,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:33,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:33,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:33,887 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:33,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:33,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:33,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:33,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743333944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:33,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:33,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:33,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:33,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:33,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743333944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:33,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:33,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743333945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:33,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:33,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:33,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:33,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:33,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:33,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:33,965 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/c9d1d7a712e34571b41d75a7bbecbd63 2024-11-27T21:34:33,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:33,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743333964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:33,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:33,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:33,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:33,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:33,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743333964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:33,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:33,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:33,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:33,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/f779a87105f4488eaef658088fe1264b is 50, key is test_row_0/B:col10/1732743272795/Put/seqid=0 2024-11-27T21:34:34,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741844_1020 (size=12001) 2024-11-27T21:34:34,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T21:34:34,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:34,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:34,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:34,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:34,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:34,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:34,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T21:34:34,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:34,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:34,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:34,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:34,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:34,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:34,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T21:34:34,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:34,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743334264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:34,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:34,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743334265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:34,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:34,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:34,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:34,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:34,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743334265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:34,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:34,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:34,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:34,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:34,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:34,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:34,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:34,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743334284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:34,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:34,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:34,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:34,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743334284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:34,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:34,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:34,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:34,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:34,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T21:34:34,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:34,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:34,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:34,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:34,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:34,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:34,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/f779a87105f4488eaef658088fe1264b 2024-11-27T21:34:34,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/59ac1624539f457fb3bc71b3cd465c81 is 50, key is test_row_0/C:col10/1732743272795/Put/seqid=0 2024-11-27T21:34:34,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741845_1021 (size=12001) 2024-11-27T21:34:34,469 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/59ac1624539f457fb3bc71b3cd465c81 2024-11-27T21:34:34,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/c9d1d7a712e34571b41d75a7bbecbd63 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/c9d1d7a712e34571b41d75a7bbecbd63 2024-11-27T21:34:34,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/c9d1d7a712e34571b41d75a7bbecbd63, entries=250, sequenceid=53, filesize=16.3 K 2024-11-27T21:34:34,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/f779a87105f4488eaef658088fe1264b as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/f779a87105f4488eaef658088fe1264b 2024-11-27T21:34:34,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T21:34:34,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:34,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:34,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:34,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:34,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:34,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:34,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/f779a87105f4488eaef658088fe1264b, entries=150, sequenceid=53, filesize=11.7 K 2024-11-27T21:34:34,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/59ac1624539f457fb3bc71b3cd465c81 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/59ac1624539f457fb3bc71b3cd465c81 2024-11-27T21:34:34,548 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/59ac1624539f457fb3bc71b3cd465c81, entries=150, sequenceid=53, filesize=11.7 K 2024-11-27T21:34:34,550 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for ebbbf218e8d25371b122092896fed576 in 1050ms, sequenceid=53, compaction requested=true 2024-11-27T21:34:34,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:34,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebbbf218e8d25371b122092896fed576:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:34:34,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:34:34,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebbbf218e8d25371b122092896fed576:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:34:34,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:34:34,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebbbf218e8d25371b122092896fed576:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:34:34,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-27T21:34:34,559 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:34:34,560 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:34:34,563 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:34:34,565 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): ebbbf218e8d25371b122092896fed576/B is initiating minor compaction (all files) 2024-11-27T21:34:34,565 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:34:34,565 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): ebbbf218e8d25371b122092896fed576/A is initiating minor compaction (all files) 2024-11-27T21:34:34,565 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebbbf218e8d25371b122092896fed576/B in TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:34,565 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebbbf218e8d25371b122092896fed576/A in TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:34,566 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/7bf3630f62bd49e6aa1d034d1c399c7c, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/fdf1aa591d674cf7a40fbc37e1fd5ce9, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/f779a87105f4488eaef658088fe1264b] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp, totalSize=35.2 K 2024-11-27T21:34:34,566 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/c3077b7e16ee482baf5e1d6b0b78321c, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/612917100cf9424a8c9e224ff7d205a6, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/c9d1d7a712e34571b41d75a7bbecbd63] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp, totalSize=42.0 K 2024-11-27T21:34:34,568 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bf3630f62bd49e6aa1d034d1c399c7c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732743272119 2024-11-27T21:34:34,569 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting fdf1aa591d674cf7a40fbc37e1fd5ce9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1732743272361 2024-11-27T21:34:34,569 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3077b7e16ee482baf5e1d6b0b78321c, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732743272119 2024-11-27T21:34:34,570 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting f779a87105f4488eaef658088fe1264b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732743272795 2024-11-27T21:34:34,570 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 612917100cf9424a8c9e224ff7d205a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1732743272361 2024-11-27T21:34:34,572 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9d1d7a712e34571b41d75a7bbecbd63, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732743272795 2024-11-27T21:34:34,610 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebbbf218e8d25371b122092896fed576#B#compaction#9 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:34:34,611 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/90cad7c46e874928b6829a87844615fa is 50, key is test_row_0/B:col10/1732743272795/Put/seqid=0 2024-11-27T21:34:34,629 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebbbf218e8d25371b122092896fed576#A#compaction#10 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:34:34,630 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/004d1554d55e46cf95816377a9fbcc5a is 50, key is test_row_0/A:col10/1732743272795/Put/seqid=0 2024-11-27T21:34:34,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741846_1022 (size=12104) 2024-11-27T21:34:34,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T21:34:34,671 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/90cad7c46e874928b6829a87844615fa as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/90cad7c46e874928b6829a87844615fa 2024-11-27T21:34:34,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:34,671 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing ebbbf218e8d25371b122092896fed576 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-27T21:34:34,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741847_1023 (size=12104) 2024-11-27T21:34:34,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/8817a5a2e7e54398a2234500389c3b6a is 50, key is test_row_0/A:col10/1732743273606/Put/seqid=0 2024-11-27T21:34:34,696 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/004d1554d55e46cf95816377a9fbcc5a as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/004d1554d55e46cf95816377a9fbcc5a 2024-11-27T21:34:34,718 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebbbf218e8d25371b122092896fed576/A of ebbbf218e8d25371b122092896fed576 into 004d1554d55e46cf95816377a9fbcc5a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:34:34,718 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:34,719 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebbbf218e8d25371b122092896fed576/B of ebbbf218e8d25371b122092896fed576 into 90cad7c46e874928b6829a87844615fa(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:34:34,719 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:34,721 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., storeName=ebbbf218e8d25371b122092896fed576/A, priority=13, startTime=1732743274552; duration=0sec 2024-11-27T21:34:34,722 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:34:34,722 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., storeName=ebbbf218e8d25371b122092896fed576/B, priority=13, startTime=1732743274555; duration=0sec 2024-11-27T21:34:34,722 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebbbf218e8d25371b122092896fed576:A 2024-11-27T21:34:34,722 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:34:34,723 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:34:34,723 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebbbf218e8d25371b122092896fed576:B 2024-11-27T21:34:34,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741848_1024 (size=12001) 2024-11-27T21:34:34,727 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/8817a5a2e7e54398a2234500389c3b6a 2024-11-27T21:34:34,737 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:34:34,737 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): ebbbf218e8d25371b122092896fed576/C is initiating minor compaction (all files) 2024-11-27T21:34:34,737 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebbbf218e8d25371b122092896fed576/C in TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:34,737 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/b3a2393c3b6f41f6a902cf18a68a0d44, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/c47faff8a40741549bb282403c543b15, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/59ac1624539f457fb3bc71b3cd465c81] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp, totalSize=35.2 K 2024-11-27T21:34:34,738 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3a2393c3b6f41f6a902cf18a68a0d44, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732743272119 2024-11-27T21:34:34,740 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting c47faff8a40741549bb282403c543b15, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1732743272361 2024-11-27T21:34:34,741 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59ac1624539f457fb3bc71b3cd465c81, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732743272795 2024-11-27T21:34:34,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/81cdaf13322849e1b17bd21bd1d9f142 is 50, key is test_row_0/B:col10/1732743273606/Put/seqid=0 2024-11-27T21:34:34,774 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebbbf218e8d25371b122092896fed576#C#compaction#13 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:34:34,775 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/8587652b7d5c49e8a3991f4ba9473aad is 50, key is test_row_0/C:col10/1732743272795/Put/seqid=0 2024-11-27T21:34:34,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741849_1025 (size=12001) 2024-11-27T21:34:34,785 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/81cdaf13322849e1b17bd21bd1d9f142 2024-11-27T21:34:34,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:34,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on ebbbf218e8d25371b122092896fed576 2024-11-27T21:34:34,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741850_1026 (size=12104) 2024-11-27T21:34:34,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:34,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743334819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:34,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:34,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743334822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:34,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:34,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:34,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:34,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:34,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:34,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:34,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/10751b3d4b944776b7cea3c716027319 is 50, key is test_row_0/C:col10/1732743273606/Put/seqid=0 2024-11-27T21:34:34,834 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/8587652b7d5c49e8a3991f4ba9473aad as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/8587652b7d5c49e8a3991f4ba9473aad 2024-11-27T21:34:34,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:34,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743334825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:34,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:34,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:34,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:34,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743334827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:34,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:34,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:34,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:34,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:34,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:34,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743334824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:34,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:34,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:34,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:34,850 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebbbf218e8d25371b122092896fed576/C of ebbbf218e8d25371b122092896fed576 into 8587652b7d5c49e8a3991f4ba9473aad(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:34:34,850 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:34,850 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., storeName=ebbbf218e8d25371b122092896fed576/C, priority=13, startTime=1732743274556; duration=0sec 2024-11-27T21:34:34,852 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:34:34,852 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebbbf218e8d25371b122092896fed576:C 2024-11-27T21:34:34,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741851_1027 (size=12001) 2024-11-27T21:34:34,873 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/10751b3d4b944776b7cea3c716027319 2024-11-27T21:34:34,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/8817a5a2e7e54398a2234500389c3b6a as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/8817a5a2e7e54398a2234500389c3b6a 2024-11-27T21:34:34,908 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/8817a5a2e7e54398a2234500389c3b6a, entries=150, sequenceid=76, filesize=11.7 K 2024-11-27T21:34:34,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/81cdaf13322849e1b17bd21bd1d9f142 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/81cdaf13322849e1b17bd21bd1d9f142 2024-11-27T21:34:34,927 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/81cdaf13322849e1b17bd21bd1d9f142, entries=150, sequenceid=76, filesize=11.7 K 2024-11-27T21:34:34,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/10751b3d4b944776b7cea3c716027319 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/10751b3d4b944776b7cea3c716027319 2024-11-27T21:34:34,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:34,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743334934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:34,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:34,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:34,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:34,944 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/10751b3d4b944776b7cea3c716027319, entries=150, sequenceid=76, filesize=11.7 K 2024-11-27T21:34:34,947 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=80.51 KB/82440 for ebbbf218e8d25371b122092896fed576 in 275ms, sequenceid=76, compaction requested=false 2024-11-27T21:34:34,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:34,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:34,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-27T21:34:34,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-27T21:34:34,956 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:34:34,956 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ebbbf218e8d25371b122092896fed576 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-27T21:34:34,956 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-27T21:34:34,956 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8940 sec 2024-11-27T21:34:34,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on ebbbf218e8d25371b122092896fed576 2024-11-27T21:34:34,962 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.9140 sec 2024-11-27T21:34:34,968 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/64ea678994cf4529a5396c4066236b79 is 50, key is test_row_0/A:col10/1732743274817/Put/seqid=0 2024-11-27T21:34:34,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741852_1028 (size=14341) 2024-11-27T21:34:35,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:35,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743335002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:35,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:35,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:35,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:35,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:35,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743335002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:35,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:35,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:35,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:35,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:35,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743335004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:35,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:35,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:35,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:35,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:35,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743335012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:35,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:35,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:35,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:35,128 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-27T21:34:35,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:35,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743335124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:35,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:35,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:35,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:35,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:35,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:35,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743335125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:35,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743335124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:35,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:35,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743335125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:35,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:35,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:35,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:35,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:35,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:35,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:35,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:35,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:35,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:35,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:35,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743335143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:35,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:35,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:35,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:35,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:35,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743335333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:35,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:35,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:35,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:35,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:35,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743335344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:35,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:35,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:35,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:35,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:35,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743335344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:35,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:35,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:35,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:35,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:35,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743335344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:35,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:35,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:35,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:35,400 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/64ea678994cf4529a5396c4066236b79 2024-11-27T21:34:35,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/aeb3a246cddc40d0bcd902aab1e9ee9e is 50, key is test_row_0/B:col10/1732743274817/Put/seqid=0 2024-11-27T21:34:35,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741853_1029 (size=12001) 2024-11-27T21:34:35,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/aeb3a246cddc40d0bcd902aab1e9ee9e 2024-11-27T21:34:35,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:35,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743335454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:35,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:35,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:35,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:35,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/739ed682efff463ba1d40d94e12cc31e is 50, key is test_row_0/C:col10/1732743274817/Put/seqid=0 2024-11-27T21:34:35,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741854_1030 (size=12001) 2024-11-27T21:34:35,506 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/739ed682efff463ba1d40d94e12cc31e 2024-11-27T21:34:35,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/64ea678994cf4529a5396c4066236b79 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/64ea678994cf4529a5396c4066236b79 2024-11-27T21:34:35,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/64ea678994cf4529a5396c4066236b79, entries=200, sequenceid=95, filesize=14.0 K 2024-11-27T21:34:35,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/aeb3a246cddc40d0bcd902aab1e9ee9e as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/aeb3a246cddc40d0bcd902aab1e9ee9e 2024-11-27T21:34:35,554 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/aeb3a246cddc40d0bcd902aab1e9ee9e, entries=150, sequenceid=95, filesize=11.7 K 2024-11-27T21:34:35,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/739ed682efff463ba1d40d94e12cc31e as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/739ed682efff463ba1d40d94e12cc31e 2024-11-27T21:34:35,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/739ed682efff463ba1d40d94e12cc31e, entries=150, sequenceid=95, filesize=11.7 K 2024-11-27T21:34:35,573 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for ebbbf218e8d25371b122092896fed576 in 617ms, sequenceid=95, compaction requested=true 2024-11-27T21:34:35,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:35,573 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:34:35,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebbbf218e8d25371b122092896fed576:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:34:35,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:34:35,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebbbf218e8d25371b122092896fed576:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:34:35,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:34:35,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebbbf218e8d25371b122092896fed576:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:34:35,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-27T21:34:35,576 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:34:35,576 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): ebbbf218e8d25371b122092896fed576/A is initiating minor compaction (all files) 2024-11-27T21:34:35,576 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebbbf218e8d25371b122092896fed576/A in TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:35,576 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:34:35,576 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/004d1554d55e46cf95816377a9fbcc5a, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/8817a5a2e7e54398a2234500389c3b6a, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/64ea678994cf4529a5396c4066236b79] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp, totalSize=37.5 K 2024-11-27T21:34:35,578 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 004d1554d55e46cf95816377a9fbcc5a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732743272795 2024-11-27T21:34:35,579 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8817a5a2e7e54398a2234500389c3b6a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732743273589 2024-11-27T21:34:35,580 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:34:35,580 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): ebbbf218e8d25371b122092896fed576/B is initiating minor compaction (all files) 2024-11-27T21:34:35,580 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebbbf218e8d25371b122092896fed576/B in TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:35,580 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/90cad7c46e874928b6829a87844615fa, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/81cdaf13322849e1b17bd21bd1d9f142, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/aeb3a246cddc40d0bcd902aab1e9ee9e] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp, totalSize=35.3 K 2024-11-27T21:34:35,581 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64ea678994cf4529a5396c4066236b79, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732743274817 2024-11-27T21:34:35,582 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 90cad7c46e874928b6829a87844615fa, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732743272795 2024-11-27T21:34:35,582 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 81cdaf13322849e1b17bd21bd1d9f142, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732743273589 2024-11-27T21:34:35,584 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting aeb3a246cddc40d0bcd902aab1e9ee9e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732743274817 2024-11-27T21:34:35,620 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebbbf218e8d25371b122092896fed576#A#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:34:35,621 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/a87d9a4b53644c68815732fad1a4bd15 is 50, key is test_row_0/A:col10/1732743274817/Put/seqid=0 2024-11-27T21:34:35,627 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebbbf218e8d25371b122092896fed576#B#compaction#19 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:34:35,627 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/81d3d073647d461c8c5b3c7a9a06230f is 50, key is test_row_0/B:col10/1732743274817/Put/seqid=0 2024-11-27T21:34:35,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741855_1031 (size=12207) 2024-11-27T21:34:35,655 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/a87d9a4b53644c68815732fad1a4bd15 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/a87d9a4b53644c68815732fad1a4bd15 2024-11-27T21:34:35,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on ebbbf218e8d25371b122092896fed576 2024-11-27T21:34:35,658 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:34:35,659 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ebbbf218e8d25371b122092896fed576 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-27T21:34:35,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741856_1032 (size=12207) 2024-11-27T21:34:35,671 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebbbf218e8d25371b122092896fed576/A of ebbbf218e8d25371b122092896fed576 into a87d9a4b53644c68815732fad1a4bd15(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:34:35,671 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:35,671 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., storeName=ebbbf218e8d25371b122092896fed576/A, priority=13, startTime=1732743275573; duration=0sec 2024-11-27T21:34:35,672 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:34:35,672 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebbbf218e8d25371b122092896fed576:A 2024-11-27T21:34:35,672 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:34:35,676 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:34:35,676 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): ebbbf218e8d25371b122092896fed576/C is initiating minor compaction (all files) 2024-11-27T21:34:35,676 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebbbf218e8d25371b122092896fed576/C in TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:35,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/5a769eb8bd604ce482058c8f86e67f81 is 50, key is test_row_0/A:col10/1732743274997/Put/seqid=0 2024-11-27T21:34:35,677 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/8587652b7d5c49e8a3991f4ba9473aad, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/10751b3d4b944776b7cea3c716027319, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/739ed682efff463ba1d40d94e12cc31e] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp, totalSize=35.3 K 2024-11-27T21:34:35,678 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8587652b7d5c49e8a3991f4ba9473aad, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732743272795 2024-11-27T21:34:35,680 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10751b3d4b944776b7cea3c716027319, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732743273589 2024-11-27T21:34:35,681 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 739ed682efff463ba1d40d94e12cc31e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732743274817 2024-11-27T21:34:35,684 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/81d3d073647d461c8c5b3c7a9a06230f as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/81d3d073647d461c8c5b3c7a9a06230f 2024-11-27T21:34:35,700 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebbbf218e8d25371b122092896fed576/B of ebbbf218e8d25371b122092896fed576 into 81d3d073647d461c8c5b3c7a9a06230f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:34:35,701 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:35,701 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., storeName=ebbbf218e8d25371b122092896fed576/B, priority=13, startTime=1732743275574; duration=0sec 2024-11-27T21:34:35,701 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:34:35,701 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebbbf218e8d25371b122092896fed576:B 2024-11-27T21:34:35,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:35,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743335698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:35,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:35,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:35,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:35,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:35,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743335700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:35,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:35,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:35,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:35,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:35,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743335701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:35,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:35,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:35,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:35,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:35,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743335704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:35,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:35,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:35,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:35,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741857_1033 (size=14341) 2024-11-27T21:34:35,711 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/5a769eb8bd604ce482058c8f86e67f81 2024-11-27T21:34:35,726 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebbbf218e8d25371b122092896fed576#C#compaction#21 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:34:35,727 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/986a22de11a24b168a30ea2c66448fa5 is 50, key is test_row_0/C:col10/1732743274817/Put/seqid=0 2024-11-27T21:34:35,727 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/9f43213a8a9d474596add0e87a544242 is 50, key is test_row_0/B:col10/1732743274997/Put/seqid=0 2024-11-27T21:34:35,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741858_1034 (size=12207) 2024-11-27T21:34:35,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741859_1035 (size=12001) 2024-11-27T21:34:35,755 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/9f43213a8a9d474596add0e87a544242 2024-11-27T21:34:35,769 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/986a22de11a24b168a30ea2c66448fa5 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/986a22de11a24b168a30ea2c66448fa5 2024-11-27T21:34:35,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/d334176494d649dfa1fc4cc31b9ab872 is 50, key is test_row_0/C:col10/1732743274997/Put/seqid=0 2024-11-27T21:34:35,785 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebbbf218e8d25371b122092896fed576/C of ebbbf218e8d25371b122092896fed576 into 986a22de11a24b168a30ea2c66448fa5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:34:35,786 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:35,786 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., storeName=ebbbf218e8d25371b122092896fed576/C, priority=13, startTime=1732743275574; duration=0sec 2024-11-27T21:34:35,786 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:34:35,786 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebbbf218e8d25371b122092896fed576:C 2024-11-27T21:34:35,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741860_1036 (size=12001) 2024-11-27T21:34:35,797 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/d334176494d649dfa1fc4cc31b9ab872 2024-11-27T21:34:35,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/5a769eb8bd604ce482058c8f86e67f81 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/5a769eb8bd604ce482058c8f86e67f81 2024-11-27T21:34:35,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:35,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743335814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:35,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:35,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:35,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:35,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:35,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743335814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:35,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:35,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:35,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:35,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:35,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743335815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:35,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:35,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743335815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:35,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:35,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:35,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:35,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:35,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:35,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:35,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/5a769eb8bd604ce482058c8f86e67f81, entries=200, sequenceid=118, filesize=14.0 K 2024-11-27T21:34:35,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/9f43213a8a9d474596add0e87a544242 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/9f43213a8a9d474596add0e87a544242 2024-11-27T21:34:35,836 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T21:34:35,838 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/9f43213a8a9d474596add0e87a544242, entries=150, sequenceid=118, filesize=11.7 K 2024-11-27T21:34:35,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/d334176494d649dfa1fc4cc31b9ab872 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/d334176494d649dfa1fc4cc31b9ab872 2024-11-27T21:34:35,850 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/d334176494d649dfa1fc4cc31b9ab872, entries=150, sequenceid=118, filesize=11.7 K 2024-11-27T21:34:35,852 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for ebbbf218e8d25371b122092896fed576 in 194ms, sequenceid=118, compaction requested=false 2024-11-27T21:34:35,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:35,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on ebbbf218e8d25371b122092896fed576 2024-11-27T21:34:35,978 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:34:35,979 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ebbbf218e8d25371b122092896fed576 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-27T21:34:35,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/eae2c79801574a07916fcfac427d583b is 50, key is test_row_0/A:col10/1732743275680/Put/seqid=0 2024-11-27T21:34:36,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741861_1037 (size=12101) 2024-11-27T21:34:36,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/eae2c79801574a07916fcfac427d583b 2024-11-27T21:34:36,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/efe1a86a1a1e4ff4898b95a77bc214b1 is 50, key is test_row_0/B:col10/1732743275680/Put/seqid=0 2024-11-27T21:34:36,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741862_1038 (size=12101) 2024-11-27T21:34:36,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/efe1a86a1a1e4ff4898b95a77bc214b1 2024-11-27T21:34:36,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/37faccf655c842b9847968f35cd5ecc4 is 50, key is test_row_0/C:col10/1732743275680/Put/seqid=0 2024-11-27T21:34:36,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741863_1039 (size=12101) 2024-11-27T21:34:36,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:36,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743336053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:36,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:36,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743336066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:36,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:36,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:36,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743336110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:36,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:36,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:36,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:36,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743336111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:36,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:36,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:36,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743336118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:36,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:36,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T21:34:36,224 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:34:36,226 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:34:36,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:36,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743336225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:36,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-27T21:34:36,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:36,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:36,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743336225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:36,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:36,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743336225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:36,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:36,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:36,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-27T21:34:36,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743336225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:36,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,231 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:34:36,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:36,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:36,232 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:34:36,232 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:34:36,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:36,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743336234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:36,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:36,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-27T21:34:36,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-27T21:34:36,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:36,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:36,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:36,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:36,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:36,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:36,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:36,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743336434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:36,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:36,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:36,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743336434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:36,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:36,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:36,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743336434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:36,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:36,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:36,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743336443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:36,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:36,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:36,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743336454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:36,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:36,494 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/37faccf655c842b9847968f35cd5ecc4 2024-11-27T21:34:36,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/eae2c79801574a07916fcfac427d583b as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/eae2c79801574a07916fcfac427d583b 2024-11-27T21:34:36,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/eae2c79801574a07916fcfac427d583b, entries=150, sequenceid=135, filesize=11.8 K 2024-11-27T21:34:36,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/efe1a86a1a1e4ff4898b95a77bc214b1 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/efe1a86a1a1e4ff4898b95a77bc214b1 2024-11-27T21:34:36,539 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/efe1a86a1a1e4ff4898b95a77bc214b1, entries=150, sequenceid=135, filesize=11.8 K 2024-11-27T21:34:36,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/37faccf655c842b9847968f35cd5ecc4 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/37faccf655c842b9847968f35cd5ecc4 2024-11-27T21:34:36,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-27T21:34:36,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-27T21:34:36,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:36,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:36,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:36,549 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:36,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:36,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/37faccf655c842b9847968f35cd5ecc4, entries=150, sequenceid=135, filesize=11.8 K 2024-11-27T21:34:36,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:36,553 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for ebbbf218e8d25371b122092896fed576 in 574ms, sequenceid=135, compaction requested=true 2024-11-27T21:34:36,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:36,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebbbf218e8d25371b122092896fed576:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:34:36,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:34:36,553 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:34:36,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebbbf218e8d25371b122092896fed576:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:34:36,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:34:36,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebbbf218e8d25371b122092896fed576:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:34:36,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-27T21:34:36,553 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:34:36,558 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:34:36,558 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38649 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:34:36,558 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): ebbbf218e8d25371b122092896fed576/B is initiating minor compaction (all files) 2024-11-27T21:34:36,558 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): ebbbf218e8d25371b122092896fed576/A is initiating minor compaction (all files) 2024-11-27T21:34:36,558 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebbbf218e8d25371b122092896fed576/B in TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:36,558 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebbbf218e8d25371b122092896fed576/A in TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:36,558 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/81d3d073647d461c8c5b3c7a9a06230f, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/9f43213a8a9d474596add0e87a544242, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/efe1a86a1a1e4ff4898b95a77bc214b1] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp, totalSize=35.5 K 2024-11-27T21:34:36,558 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/a87d9a4b53644c68815732fad1a4bd15, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/5a769eb8bd604ce482058c8f86e67f81, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/eae2c79801574a07916fcfac427d583b] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp, totalSize=37.7 K 2024-11-27T21:34:36,561 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting a87d9a4b53644c68815732fad1a4bd15, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732743274817 2024-11-27T21:34:36,561 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81d3d073647d461c8c5b3c7a9a06230f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732743274817 2024-11-27T21:34:36,562 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a769eb8bd604ce482058c8f86e67f81, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732743274997 2024-11-27T21:34:36,562 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f43213a8a9d474596add0e87a544242, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732743274997 2024-11-27T21:34:36,562 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting eae2c79801574a07916fcfac427d583b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732743275680 2024-11-27T21:34:36,563 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting efe1a86a1a1e4ff4898b95a77bc214b1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732743275680 2024-11-27T21:34:36,588 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebbbf218e8d25371b122092896fed576#A#compaction#27 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:34:36,590 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebbbf218e8d25371b122092896fed576#B#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:34:36,590 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/f127cd9d03db4d3f848e9e61531160b5 is 50, key is test_row_0/A:col10/1732743275680/Put/seqid=0 2024-11-27T21:34:36,591 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/f260d87f55eb43229f8f6083a591cf18 is 50, key is test_row_0/B:col10/1732743275680/Put/seqid=0 2024-11-27T21:34:36,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741864_1040 (size=12409) 2024-11-27T21:34:36,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741865_1041 (size=12409) 2024-11-27T21:34:36,648 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/f127cd9d03db4d3f848e9e61531160b5 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/f127cd9d03db4d3f848e9e61531160b5 2024-11-27T21:34:36,670 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/f260d87f55eb43229f8f6083a591cf18 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/f260d87f55eb43229f8f6083a591cf18 2024-11-27T21:34:36,671 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebbbf218e8d25371b122092896fed576/A of ebbbf218e8d25371b122092896fed576 into f127cd9d03db4d3f848e9e61531160b5(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:34:36,671 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:36,671 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., storeName=ebbbf218e8d25371b122092896fed576/A, priority=13, startTime=1732743276553; duration=0sec 2024-11-27T21:34:36,675 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:34:36,675 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebbbf218e8d25371b122092896fed576:A 2024-11-27T21:34:36,676 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:34:36,680 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:34:36,681 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): ebbbf218e8d25371b122092896fed576/C is initiating minor compaction (all files) 2024-11-27T21:34:36,681 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebbbf218e8d25371b122092896fed576/C in TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:36,681 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/986a22de11a24b168a30ea2c66448fa5, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/d334176494d649dfa1fc4cc31b9ab872, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/37faccf655c842b9847968f35cd5ecc4] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp, totalSize=35.5 K 2024-11-27T21:34:36,680 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebbbf218e8d25371b122092896fed576/B of ebbbf218e8d25371b122092896fed576 into f260d87f55eb43229f8f6083a591cf18(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:34:36,681 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:36,681 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., storeName=ebbbf218e8d25371b122092896fed576/B, priority=13, startTime=1732743276553; duration=0sec 2024-11-27T21:34:36,681 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:34:36,681 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebbbf218e8d25371b122092896fed576:B 2024-11-27T21:34:36,683 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 986a22de11a24b168a30ea2c66448fa5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732743274817 2024-11-27T21:34:36,684 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting d334176494d649dfa1fc4cc31b9ab872, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732743274997 2024-11-27T21:34:36,684 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 37faccf655c842b9847968f35cd5ecc4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732743275680 2024-11-27T21:34:36,698 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-27T21:34:36,699 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-27T21:34:36,701 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-27T21:34:36,701 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-27T21:34:36,701 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-27T21:34:36,701 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-27T21:34:36,702 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-27T21:34:36,703 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-27T21:34:36,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-27T21:34:36,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:36,706 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing ebbbf218e8d25371b122092896fed576 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-27T21:34:36,716 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebbbf218e8d25371b122092896fed576#C#compaction#29 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:34:36,716 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/ecc6f2840bd94670809c7b51966d109a is 50, key is test_row_0/C:col10/1732743275680/Put/seqid=0 2024-11-27T21:34:36,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/806b189670194c978c0c97961f271295 is 50, key is test_row_0/A:col10/1732743276065/Put/seqid=0 2024-11-27T21:34:36,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741866_1042 (size=12409) 2024-11-27T21:34:36,745 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/ecc6f2840bd94670809c7b51966d109a as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/ecc6f2840bd94670809c7b51966d109a 2024-11-27T21:34:36,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741867_1043 (size=12151) 2024-11-27T21:34:36,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:36,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on ebbbf218e8d25371b122092896fed576 2024-11-27T21:34:36,754 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/806b189670194c978c0c97961f271295 2024-11-27T21:34:36,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/6e88e6aea7184e489de554d3c51ff664 is 50, key is test_row_0/B:col10/1732743276065/Put/seqid=0 2024-11-27T21:34:36,777 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebbbf218e8d25371b122092896fed576/C of ebbbf218e8d25371b122092896fed576 into ecc6f2840bd94670809c7b51966d109a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:34:36,777 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:36,778 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., storeName=ebbbf218e8d25371b122092896fed576/C, priority=13, startTime=1732743276553; duration=0sec 2024-11-27T21:34:36,779 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:34:36,779 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebbbf218e8d25371b122092896fed576:C 2024-11-27T21:34:36,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:36,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743336769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:36,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:36,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:36,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743336770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:36,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:36,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:36,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743336771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:36,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:36,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743336773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:36,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:36,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:36,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743336773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:36,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:36,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:36,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741868_1044 (size=12151) 2024-11-27T21:34:36,790 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/6e88e6aea7184e489de554d3c51ff664 2024-11-27T21:34:36,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/bf60ddaa82bb40a6991f517008d5f981 is 50, key is test_row_0/C:col10/1732743276065/Put/seqid=0 2024-11-27T21:34:36,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741869_1045 (size=12151) 2024-11-27T21:34:36,832 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/bf60ddaa82bb40a6991f517008d5f981 2024-11-27T21:34:36,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/806b189670194c978c0c97961f271295 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/806b189670194c978c0c97961f271295 2024-11-27T21:34:36,853 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/806b189670194c978c0c97961f271295, entries=150, sequenceid=160, filesize=11.9 K 2024-11-27T21:34:36,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-27T21:34:36,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/6e88e6aea7184e489de554d3c51ff664 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/6e88e6aea7184e489de554d3c51ff664 2024-11-27T21:34:36,871 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/6e88e6aea7184e489de554d3c51ff664, entries=150, sequenceid=160, filesize=11.9 K 2024-11-27T21:34:36,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/bf60ddaa82bb40a6991f517008d5f981 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/bf60ddaa82bb40a6991f517008d5f981 2024-11-27T21:34:36,887 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/bf60ddaa82bb40a6991f517008d5f981, entries=150, sequenceid=160, filesize=11.9 K 2024-11-27T21:34:36,889 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for ebbbf218e8d25371b122092896fed576 in 183ms, sequenceid=160, compaction requested=false 2024-11-27T21:34:36,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:36,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:36,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-27T21:34:36,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-27T21:34:36,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on ebbbf218e8d25371b122092896fed576 2024-11-27T21:34:36,896 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:34:36,897 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ebbbf218e8d25371b122092896fed576 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-27T21:34:36,899 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-27T21:34:36,899 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 660 msec 2024-11-27T21:34:36,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/ad08726af6c94411a101ece386e7966e is 50, key is test_row_0/A:col10/1732743276766/Put/seqid=0 2024-11-27T21:34:36,907 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 674 msec 2024-11-27T21:34:36,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741870_1046 (size=14541) 2024-11-27T21:34:36,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/ad08726af6c94411a101ece386e7966e 2024-11-27T21:34:36,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/4557ee9ad46240a7abf2d005696d0eb2 is 50, key is test_row_0/B:col10/1732743276766/Put/seqid=0 2024-11-27T21:34:36,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741871_1047 (size=12151) 2024-11-27T21:34:36,947 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/4557ee9ad46240a7abf2d005696d0eb2 2024-11-27T21:34:36,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:36,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:36,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743336937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:36,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743336938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:36,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:36,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:36,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:36,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743336946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:36,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:36,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:36,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743336952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:36,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:36,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:36,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743336952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:36,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:36,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:36,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/cd2d5a09148547eab02e01f305a5d0fc is 50, key is test_row_0/C:col10/1732743276766/Put/seqid=0 2024-11-27T21:34:36,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741872_1048 (size=12151) 2024-11-27T21:34:37,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:37,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743337064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:37,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:37,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743337064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:37,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:37,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:37,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743337065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:37,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:37,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:37,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:37,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743337066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:37,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:37,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743337066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:37,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:37,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:37,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:37,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:37,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743337274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:37,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743337274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:37,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:37,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743337274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:37,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:37,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:37,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:37,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743337274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:37,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:37,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:37,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743337274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:37,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:37,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:37,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-27T21:34:37,363 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:34:37,364 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:34:37,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-27T21:34:37,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-27T21:34:37,367 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:34:37,368 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:34:37,368 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:34:37,383 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/cd2d5a09148547eab02e01f305a5d0fc 2024-11-27T21:34:37,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/ad08726af6c94411a101ece386e7966e as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/ad08726af6c94411a101ece386e7966e 2024-11-27T21:34:37,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/ad08726af6c94411a101ece386e7966e, entries=200, sequenceid=176, filesize=14.2 K 2024-11-27T21:34:37,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/4557ee9ad46240a7abf2d005696d0eb2 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/4557ee9ad46240a7abf2d005696d0eb2 2024-11-27T21:34:37,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/4557ee9ad46240a7abf2d005696d0eb2, entries=150, sequenceid=176, filesize=11.9 K 2024-11-27T21:34:37,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/cd2d5a09148547eab02e01f305a5d0fc as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/cd2d5a09148547eab02e01f305a5d0fc 2024-11-27T21:34:37,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/cd2d5a09148547eab02e01f305a5d0fc, entries=150, sequenceid=176, filesize=11.9 K 2024-11-27T21:34:37,427 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for ebbbf218e8d25371b122092896fed576 in 531ms, sequenceid=176, compaction requested=true 2024-11-27T21:34:37,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:37,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebbbf218e8d25371b122092896fed576:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:34:37,427 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:34:37,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:34:37,427 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:34:37,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebbbf218e8d25371b122092896fed576:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:34:37,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:34:37,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebbbf218e8d25371b122092896fed576:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:34:37,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:34:37,431 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39101 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:34:37,431 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:34:37,431 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): ebbbf218e8d25371b122092896fed576/A is initiating minor compaction (all files) 2024-11-27T21:34:37,431 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebbbf218e8d25371b122092896fed576/A in TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:37,431 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): ebbbf218e8d25371b122092896fed576/B is initiating minor compaction (all files) 2024-11-27T21:34:37,431 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebbbf218e8d25371b122092896fed576/B in TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:37,431 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/f127cd9d03db4d3f848e9e61531160b5, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/806b189670194c978c0c97961f271295, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/ad08726af6c94411a101ece386e7966e] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp, totalSize=38.2 K 2024-11-27T21:34:37,431 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/f260d87f55eb43229f8f6083a591cf18, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/6e88e6aea7184e489de554d3c51ff664, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/4557ee9ad46240a7abf2d005696d0eb2] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp, totalSize=35.9 K 2024-11-27T21:34:37,432 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting f127cd9d03db4d3f848e9e61531160b5, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732743275680 2024-11-27T21:34:37,432 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting f260d87f55eb43229f8f6083a591cf18, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732743275680 2024-11-27T21:34:37,432 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 806b189670194c978c0c97961f271295, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732743276048 2024-11-27T21:34:37,433 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e88e6aea7184e489de554d3c51ff664, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732743276048 2024-11-27T21:34:37,433 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad08726af6c94411a101ece386e7966e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732743276765 2024-11-27T21:34:37,433 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 4557ee9ad46240a7abf2d005696d0eb2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732743276766 2024-11-27T21:34:37,448 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebbbf218e8d25371b122092896fed576#A#compaction#36 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:34:37,448 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/1a609947b9764a819e710d7b0b8fdcf0 is 50, key is test_row_0/A:col10/1732743276766/Put/seqid=0 2024-11-27T21:34:37,459 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebbbf218e8d25371b122092896fed576#B#compaction#37 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:34:37,459 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/9c41c56cd09e4c1b8fe2f2df84579fea is 50, key is test_row_0/B:col10/1732743276766/Put/seqid=0 2024-11-27T21:34:37,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-27T21:34:37,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741874_1050 (size=12561) 2024-11-27T21:34:37,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741873_1049 (size=12561) 2024-11-27T21:34:37,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-27T21:34:37,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:37,523 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing ebbbf218e8d25371b122092896fed576 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-27T21:34:37,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/bc47f9b9b6ea442cbae08d6909963fa3 is 50, key is test_row_0/A:col10/1732743276948/Put/seqid=0 2024-11-27T21:34:37,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741875_1051 (size=12151) 2024-11-27T21:34:37,549 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/bc47f9b9b6ea442cbae08d6909963fa3 2024-11-27T21:34:37,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/e71be646b259479480da4dcfaa74e3c5 is 50, key is test_row_0/B:col10/1732743276948/Put/seqid=0 2024-11-27T21:34:37,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741876_1052 (size=12151) 2024-11-27T21:34:37,580 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/e71be646b259479480da4dcfaa74e3c5 2024-11-27T21:34:37,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on ebbbf218e8d25371b122092896fed576 2024-11-27T21:34:37,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:37,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/287361b1b5cf4d1197963bb16960d751 is 50, key is test_row_0/C:col10/1732743276948/Put/seqid=0 2024-11-27T21:34:37,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741877_1053 (size=12151) 2024-11-27T21:34:37,613 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/287361b1b5cf4d1197963bb16960d751 2024-11-27T21:34:37,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:37,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743337610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:37,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/bc47f9b9b6ea442cbae08d6909963fa3 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/bc47f9b9b6ea442cbae08d6909963fa3 2024-11-27T21:34:37,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:37,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:37,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743337612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:37,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:37,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:37,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743337616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:37,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:37,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:37,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743337619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:37,630 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/bc47f9b9b6ea442cbae08d6909963fa3, entries=150, sequenceid=198, filesize=11.9 K 2024-11-27T21:34:37,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:37,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743337621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:37,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:37,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:37,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/e71be646b259479480da4dcfaa74e3c5 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/e71be646b259479480da4dcfaa74e3c5 2024-11-27T21:34:37,646 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/e71be646b259479480da4dcfaa74e3c5, entries=150, sequenceid=198, filesize=11.9 K 2024-11-27T21:34:37,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/287361b1b5cf4d1197963bb16960d751 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/287361b1b5cf4d1197963bb16960d751 2024-11-27T21:34:37,661 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/287361b1b5cf4d1197963bb16960d751, entries=150, sequenceid=198, filesize=11.9 K 2024-11-27T21:34:37,665 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for ebbbf218e8d25371b122092896fed576 in 143ms, sequenceid=198, compaction requested=true 2024-11-27T21:34:37,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:37,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:37,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-27T21:34:37,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-27T21:34:37,671 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-27T21:34:37,671 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 300 msec 2024-11-27T21:34:37,674 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 308 msec 2024-11-27T21:34:37,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-27T21:34:37,684 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:34:37,685 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:34:37,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-27T21:34:37,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-27T21:34:37,688 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:34:37,689 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:34:37,689 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:34:37,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on ebbbf218e8d25371b122092896fed576 2024-11-27T21:34:37,746 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:34:37,746 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ebbbf218e8d25371b122092896fed576 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-27T21:34:37,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/ec70ef3627bb408ea5b767217d4c9745 is 50, key is test_row_0/A:col10/1732743277744/Put/seqid=0 2024-11-27T21:34:37,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741878_1054 (size=14541) 2024-11-27T21:34:37,773 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/ec70ef3627bb408ea5b767217d4c9745 2024-11-27T21:34:37,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:37,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743337785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:37,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:37,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743337786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:37,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:37,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:37,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743337786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:37,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:37,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:37,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/f46405dfac774be0a7d5d2d1571b9f1d is 50, key is test_row_0/B:col10/1732743277744/Put/seqid=0 2024-11-27T21:34:37,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-27T21:34:37,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:37,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743337789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:37,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:37,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743337790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:37,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:37,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:37,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741879_1055 (size=12151) 2024-11-27T21:34:37,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-27T21:34:37,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:37,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:37,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:37,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:37,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:37,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:37,899 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/1a609947b9764a819e710d7b0b8fdcf0 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/1a609947b9764a819e710d7b0b8fdcf0 2024-11-27T21:34:37,899 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/9c41c56cd09e4c1b8fe2f2df84579fea as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/9c41c56cd09e4c1b8fe2f2df84579fea 2024-11-27T21:34:37,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:37,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743337894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:37,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:37,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:37,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743337894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:37,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:37,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:37,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743337894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:37,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:37,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:37,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743337904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:37,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:37,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743337905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:37,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:37,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:37,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:37,913 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebbbf218e8d25371b122092896fed576/B of ebbbf218e8d25371b122092896fed576 into 9c41c56cd09e4c1b8fe2f2df84579fea(size=12.3 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:34:37,914 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:37,914 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., storeName=ebbbf218e8d25371b122092896fed576/B, priority=13, startTime=1732743277427; duration=0sec 2024-11-27T21:34:37,914 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebbbf218e8d25371b122092896fed576/A of ebbbf218e8d25371b122092896fed576 into 1a609947b9764a819e710d7b0b8fdcf0(size=12.3 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:34:37,914 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:37,914 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., storeName=ebbbf218e8d25371b122092896fed576/A, priority=13, startTime=1732743277427; duration=0sec 2024-11-27T21:34:37,914 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:34:37,914 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:34:37,914 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebbbf218e8d25371b122092896fed576:B 2024-11-27T21:34:37,914 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebbbf218e8d25371b122092896fed576:A 2024-11-27T21:34:37,914 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:34:37,916 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:34:37,916 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): ebbbf218e8d25371b122092896fed576/C is initiating minor compaction (all files) 2024-11-27T21:34:37,916 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebbbf218e8d25371b122092896fed576/C in TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:37,917 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/ecc6f2840bd94670809c7b51966d109a, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/bf60ddaa82bb40a6991f517008d5f981, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/cd2d5a09148547eab02e01f305a5d0fc, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/287361b1b5cf4d1197963bb16960d751] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp, totalSize=47.7 K 2024-11-27T21:34:37,918 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting ecc6f2840bd94670809c7b51966d109a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732743275680 2024-11-27T21:34:37,918 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting bf60ddaa82bb40a6991f517008d5f981, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732743276048 2024-11-27T21:34:37,919 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting cd2d5a09148547eab02e01f305a5d0fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732743276766 2024-11-27T21:34:37,920 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 287361b1b5cf4d1197963bb16960d751, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732743276921 2024-11-27T21:34:37,938 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebbbf218e8d25371b122092896fed576#C#compaction#43 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:34:37,939 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/aa73496fbadf468387fe5db4d08574d0 is 50, key is test_row_0/C:col10/1732743276948/Put/seqid=0 2024-11-27T21:34:37,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741880_1056 (size=12595) 2024-11-27T21:34:37,962 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/aa73496fbadf468387fe5db4d08574d0 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/aa73496fbadf468387fe5db4d08574d0 2024-11-27T21:34:37,972 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ebbbf218e8d25371b122092896fed576/C of ebbbf218e8d25371b122092896fed576 into aa73496fbadf468387fe5db4d08574d0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:34:37,972 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:37,972 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., storeName=ebbbf218e8d25371b122092896fed576/C, priority=12, startTime=1732743277429; duration=0sec 2024-11-27T21:34:37,973 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:34:37,973 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebbbf218e8d25371b122092896fed576:C 2024-11-27T21:34:37,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-27T21:34:37,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:37,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:37,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:37,999 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:38,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:38,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:38,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-27T21:34:38,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:38,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743338114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:38,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:38,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:38,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:38,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:38,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743338114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:38,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:38,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:38,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:38,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:38,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:38,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743338114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:38,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743338115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:38,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:38,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:38,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:38,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:38,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:38,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:38,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:38,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743338115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:38,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:38,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:38,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:38,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-27T21:34:38,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:38,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:38,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:38,154 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:38,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:38,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:38,211 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/f46405dfac774be0a7d5d2d1571b9f1d 2024-11-27T21:34:38,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/b8c0574b668e4479bc166cb5800c8058 is 50, key is test_row_0/C:col10/1732743277744/Put/seqid=0 2024-11-27T21:34:38,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741881_1057 (size=12151) 2024-11-27T21:34:38,237 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/b8c0574b668e4479bc166cb5800c8058 2024-11-27T21:34:38,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/ec70ef3627bb408ea5b767217d4c9745 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/ec70ef3627bb408ea5b767217d4c9745 2024-11-27T21:34:38,260 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/ec70ef3627bb408ea5b767217d4c9745, entries=200, sequenceid=217, filesize=14.2 K 2024-11-27T21:34:38,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/f46405dfac774be0a7d5d2d1571b9f1d as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/f46405dfac774be0a7d5d2d1571b9f1d 2024-11-27T21:34:38,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/f46405dfac774be0a7d5d2d1571b9f1d, entries=150, sequenceid=217, filesize=11.9 K 2024-11-27T21:34:38,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/b8c0574b668e4479bc166cb5800c8058 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/b8c0574b668e4479bc166cb5800c8058 2024-11-27T21:34:38,279 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/b8c0574b668e4479bc166cb5800c8058, entries=150, sequenceid=217, filesize=11.9 K 2024-11-27T21:34:38,281 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for ebbbf218e8d25371b122092896fed576 in 535ms, sequenceid=217, compaction requested=true 2024-11-27T21:34:38,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:38,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebbbf218e8d25371b122092896fed576:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:34:38,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:34:38,281 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:34:38,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebbbf218e8d25371b122092896fed576:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:34:38,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:34:38,281 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:34:38,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebbbf218e8d25371b122092896fed576:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:34:38,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:34:38,283 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:34:38,283 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:34:38,283 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): ebbbf218e8d25371b122092896fed576/A is initiating minor compaction (all files) 2024-11-27T21:34:38,283 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): ebbbf218e8d25371b122092896fed576/B is initiating minor compaction (all files) 2024-11-27T21:34:38,283 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebbbf218e8d25371b122092896fed576/A in TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:38,283 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebbbf218e8d25371b122092896fed576/B in TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:38,284 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/1a609947b9764a819e710d7b0b8fdcf0, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/bc47f9b9b6ea442cbae08d6909963fa3, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/ec70ef3627bb408ea5b767217d4c9745] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp, totalSize=38.3 K 2024-11-27T21:34:38,284 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/9c41c56cd09e4c1b8fe2f2df84579fea, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/e71be646b259479480da4dcfaa74e3c5, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/f46405dfac774be0a7d5d2d1571b9f1d] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp, totalSize=36.0 K 2024-11-27T21:34:38,284 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c41c56cd09e4c1b8fe2f2df84579fea, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732743276766 2024-11-27T21:34:38,284 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a609947b9764a819e710d7b0b8fdcf0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732743276766 2024-11-27T21:34:38,287 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting e71be646b259479480da4dcfaa74e3c5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732743276921 2024-11-27T21:34:38,287 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc47f9b9b6ea442cbae08d6909963fa3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732743276921 2024-11-27T21:34:38,287 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting f46405dfac774be0a7d5d2d1571b9f1d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732743277617 2024-11-27T21:34:38,287 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec70ef3627bb408ea5b767217d4c9745, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732743277617 2024-11-27T21:34:38,304 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebbbf218e8d25371b122092896fed576#B#compaction#45 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:34:38,305 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/9a619ac3871247a7b3663273901034d9 is 50, key is test_row_0/B:col10/1732743277744/Put/seqid=0 2024-11-27T21:34:38,305 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebbbf218e8d25371b122092896fed576#A#compaction#46 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:34:38,306 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/84975eaa4d1243afa57a32213ff34942 is 50, key is test_row_0/A:col10/1732743277744/Put/seqid=0 2024-11-27T21:34:38,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-27T21:34:38,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:38,308 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing ebbbf218e8d25371b122092896fed576 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-27T21:34:38,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-27T21:34:38,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/b6c1428f7d0f40e5aa5a54d845bbb830 is 50, key is test_row_0/A:col10/1732743277787/Put/seqid=0 2024-11-27T21:34:38,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741882_1058 (size=12663) 2024-11-27T21:34:38,342 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/9a619ac3871247a7b3663273901034d9 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/9a619ac3871247a7b3663273901034d9 2024-11-27T21:34:38,359 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebbbf218e8d25371b122092896fed576/B of ebbbf218e8d25371b122092896fed576 into 9a619ac3871247a7b3663273901034d9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:34:38,359 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:38,359 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., storeName=ebbbf218e8d25371b122092896fed576/B, priority=13, startTime=1732743278281; duration=0sec 2024-11-27T21:34:38,360 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:34:38,360 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebbbf218e8d25371b122092896fed576:B 2024-11-27T21:34:38,360 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-27T21:34:38,361 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-27T21:34:38,363 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-27T21:34:38,363 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. because compaction request was cancelled 2024-11-27T21:34:38,363 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebbbf218e8d25371b122092896fed576:C 2024-11-27T21:34:38,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741883_1059 (size=12663) 2024-11-27T21:34:38,378 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/84975eaa4d1243afa57a32213ff34942 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/84975eaa4d1243afa57a32213ff34942 2024-11-27T21:34:38,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741884_1060 (size=12151) 2024-11-27T21:34:38,382 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/b6c1428f7d0f40e5aa5a54d845bbb830 2024-11-27T21:34:38,391 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebbbf218e8d25371b122092896fed576/A of ebbbf218e8d25371b122092896fed576 into 84975eaa4d1243afa57a32213ff34942(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:34:38,391 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:38,391 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., storeName=ebbbf218e8d25371b122092896fed576/A, priority=13, startTime=1732743278281; duration=0sec 2024-11-27T21:34:38,392 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:34:38,392 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebbbf218e8d25371b122092896fed576:A 2024-11-27T21:34:38,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/045af624626a4b0ca42b73be1510a7c9 is 50, key is test_row_0/B:col10/1732743277787/Put/seqid=0 2024-11-27T21:34:38,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741885_1061 (size=12151) 2024-11-27T21:34:38,427 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/045af624626a4b0ca42b73be1510a7c9 2024-11-27T21:34:38,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on ebbbf218e8d25371b122092896fed576 2024-11-27T21:34:38,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:38,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/3f2a4ed6a9174bc98ad4a3757c4eb40a is 50, key is test_row_0/C:col10/1732743277787/Put/seqid=0 2024-11-27T21:34:38,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741886_1062 (size=12151) 2024-11-27T21:34:38,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:38,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743338483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:38,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:38,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:38,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:38,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:38,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743338484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:38,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:38,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:38,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:38,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:38,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:38,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743338484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:38,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743338487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:38,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:38,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:38,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:38,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:38,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:38,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:38,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:38,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743338485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:38,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:38,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:38,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:38,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:38,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743338604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:38,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:38,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:38,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:38,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:38,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743338604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:38,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:38,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:38,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743338604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:38,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:38,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:38,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:38,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743338604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:38,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:38,612 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:38,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743338605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:38,612 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:38,612 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:38,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:38,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:38,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:38,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:38,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:38,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:38,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:38,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743338813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:38,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:38,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:38,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:38,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-27T21:34:38,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:38,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743338824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:38,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:38,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:38,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:38,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:38,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743338824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:38,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:38,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:38,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:38,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:38,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743338825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:38,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:38,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:38,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:38,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:38,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743338825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:38,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:38,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:38,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:38,856 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/3f2a4ed6a9174bc98ad4a3757c4eb40a 2024-11-27T21:34:38,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/b6c1428f7d0f40e5aa5a54d845bbb830 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/b6c1428f7d0f40e5aa5a54d845bbb830 2024-11-27T21:34:38,871 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/b6c1428f7d0f40e5aa5a54d845bbb830, entries=150, sequenceid=238, filesize=11.9 K 2024-11-27T21:34:38,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/045af624626a4b0ca42b73be1510a7c9 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/045af624626a4b0ca42b73be1510a7c9 2024-11-27T21:34:38,894 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/045af624626a4b0ca42b73be1510a7c9, entries=150, sequenceid=238, filesize=11.9 K 2024-11-27T21:34:38,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/3f2a4ed6a9174bc98ad4a3757c4eb40a as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/3f2a4ed6a9174bc98ad4a3757c4eb40a 2024-11-27T21:34:38,901 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/3f2a4ed6a9174bc98ad4a3757c4eb40a, entries=150, sequenceid=238, filesize=11.9 K 2024-11-27T21:34:38,903 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for ebbbf218e8d25371b122092896fed576 in 595ms, sequenceid=238, compaction requested=true 2024-11-27T21:34:38,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:38,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:38,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-27T21:34:38,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-27T21:34:38,908 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-27T21:34:38,908 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2160 sec 2024-11-27T21:34:38,911 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.2240 sec 2024-11-27T21:34:39,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on ebbbf218e8d25371b122092896fed576 2024-11-27T21:34:39,126 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:34:39,126 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ebbbf218e8d25371b122092896fed576 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-27T21:34:39,133 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/dc100c800938408fa91745bd32f90639 is 50, key is test_row_0/A:col10/1732743278482/Put/seqid=0 2024-11-27T21:34:39,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741887_1063 (size=14641) 2024-11-27T21:34:39,152 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/dc100c800938408fa91745bd32f90639 2024-11-27T21:34:39,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/96a8c2a70fb74f5da971c1d12a661e98 is 50, key is test_row_0/B:col10/1732743278482/Put/seqid=0 2024-11-27T21:34:39,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:39,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743339162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:39,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:39,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:39,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:39,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:39,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743339169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:39,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:39,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:39,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:39,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:39,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743339173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:39,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:39,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:39,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:39,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:39,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743339174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:39,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:39,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743339175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:39,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:39,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:39,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:39,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:39,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:39,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:39,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741888_1064 (size=12251) 2024-11-27T21:34:39,199 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/96a8c2a70fb74f5da971c1d12a661e98 2024-11-27T21:34:39,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/a35cc849614b433c98dd01663389c1cf is 50, key is test_row_0/C:col10/1732743278482/Put/seqid=0 2024-11-27T21:34:39,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741889_1065 (size=12251) 2024-11-27T21:34:39,236 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/a35cc849614b433c98dd01663389c1cf 2024-11-27T21:34:39,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/dc100c800938408fa91745bd32f90639 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/dc100c800938408fa91745bd32f90639 2024-11-27T21:34:39,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/dc100c800938408fa91745bd32f90639, entries=200, sequenceid=259, filesize=14.3 K 2024-11-27T21:34:39,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/96a8c2a70fb74f5da971c1d12a661e98 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/96a8c2a70fb74f5da971c1d12a661e98 2024-11-27T21:34:39,264 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/96a8c2a70fb74f5da971c1d12a661e98, entries=150, sequenceid=259, filesize=12.0 K 2024-11-27T21:34:39,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/a35cc849614b433c98dd01663389c1cf as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/a35cc849614b433c98dd01663389c1cf 2024-11-27T21:34:39,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/a35cc849614b433c98dd01663389c1cf, entries=150, sequenceid=259, filesize=12.0 K 2024-11-27T21:34:39,280 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for ebbbf218e8d25371b122092896fed576 in 153ms, sequenceid=259, compaction requested=true 2024-11-27T21:34:39,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:39,280 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:34:39,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebbbf218e8d25371b122092896fed576:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:34:39,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:34:39,281 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:34:39,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebbbf218e8d25371b122092896fed576:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:34:39,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:34:39,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebbbf218e8d25371b122092896fed576:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:34:39,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:34:39,282 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39455 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:34:39,282 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): ebbbf218e8d25371b122092896fed576/A is initiating minor compaction (all files) 2024-11-27T21:34:39,282 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebbbf218e8d25371b122092896fed576/A in TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:39,282 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/84975eaa4d1243afa57a32213ff34942, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/b6c1428f7d0f40e5aa5a54d845bbb830, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/dc100c800938408fa91745bd32f90639] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp, totalSize=38.5 K 2024-11-27T21:34:39,283 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37065 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:34:39,283 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): ebbbf218e8d25371b122092896fed576/B is initiating minor compaction (all files) 2024-11-27T21:34:39,283 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebbbf218e8d25371b122092896fed576/B in TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:39,283 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/9a619ac3871247a7b3663273901034d9, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/045af624626a4b0ca42b73be1510a7c9, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/96a8c2a70fb74f5da971c1d12a661e98] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp, totalSize=36.2 K 2024-11-27T21:34:39,284 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a619ac3871247a7b3663273901034d9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732743277617 2024-11-27T21:34:39,284 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84975eaa4d1243afa57a32213ff34942, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732743277617 2024-11-27T21:34:39,286 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 045af624626a4b0ca42b73be1510a7c9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732743277784 2024-11-27T21:34:39,287 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 96a8c2a70fb74f5da971c1d12a661e98, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732743278482 2024-11-27T21:34:39,288 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6c1428f7d0f40e5aa5a54d845bbb830, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732743277784 2024-11-27T21:34:39,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on ebbbf218e8d25371b122092896fed576 2024-11-27T21:34:39,289 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc100c800938408fa91745bd32f90639, keycount=200, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732743278431 2024-11-27T21:34:39,289 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:34:39,289 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ebbbf218e8d25371b122092896fed576 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-27T21:34:39,312 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebbbf218e8d25371b122092896fed576#A#compaction#54 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:34:39,313 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/51dfcc094e9a4363ad69d5e429c4e333 is 50, key is test_row_0/A:col10/1732743278482/Put/seqid=0 2024-11-27T21:34:39,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/1e4f3bccfe0e48baba3f049078627c37 is 50, key is test_row_0/A:col10/1732743279289/Put/seqid=0 2024-11-27T21:34:39,315 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebbbf218e8d25371b122092896fed576#B#compaction#55 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:34:39,316 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/20a07468e11e4ef8bf0a1615446a0dca is 50, key is test_row_0/B:col10/1732743278482/Put/seqid=0 2024-11-27T21:34:39,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:39,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743339323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:39,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:39,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:39,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:39,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:39,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743339328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:39,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:39,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:39,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:39,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741890_1066 (size=12865) 2024-11-27T21:34:39,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741891_1067 (size=14741) 2024-11-27T21:34:39,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:39,344 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/1e4f3bccfe0e48baba3f049078627c37 2024-11-27T21:34:39,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743339329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:39,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:39,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:39,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:39,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743339330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:39,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:39,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:39,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:39,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:39,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:39,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743339331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:39,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:39,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:39,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:39,354 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/51dfcc094e9a4363ad69d5e429c4e333 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/51dfcc094e9a4363ad69d5e429c4e333 2024-11-27T21:34:39,366 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebbbf218e8d25371b122092896fed576/A of ebbbf218e8d25371b122092896fed576 into 51dfcc094e9a4363ad69d5e429c4e333(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:34:39,366 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:39,367 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., storeName=ebbbf218e8d25371b122092896fed576/A, priority=13, startTime=1732743279280; duration=0sec 2024-11-27T21:34:39,367 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:34:39,367 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebbbf218e8d25371b122092896fed576:A 2024-11-27T21:34:39,367 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:34:39,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741892_1068 (size=12865) 2024-11-27T21:34:39,372 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49148 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:34:39,372 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): ebbbf218e8d25371b122092896fed576/C is initiating minor compaction (all files) 2024-11-27T21:34:39,373 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebbbf218e8d25371b122092896fed576/C in TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:39,373 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/aa73496fbadf468387fe5db4d08574d0, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/b8c0574b668e4479bc166cb5800c8058, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/3f2a4ed6a9174bc98ad4a3757c4eb40a, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/a35cc849614b433c98dd01663389c1cf] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp, totalSize=48.0 K 2024-11-27T21:34:39,374 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa73496fbadf468387fe5db4d08574d0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732743276921 2024-11-27T21:34:39,375 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8c0574b668e4479bc166cb5800c8058, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732743277617 2024-11-27T21:34:39,375 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f2a4ed6a9174bc98ad4a3757c4eb40a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732743277784 2024-11-27T21:34:39,377 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting a35cc849614b433c98dd01663389c1cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732743278482 2024-11-27T21:34:39,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/3516d22fcb5d4e758a517537d1d3e934 is 50, key is test_row_0/B:col10/1732743279289/Put/seqid=0 2024-11-27T21:34:39,379 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/20a07468e11e4ef8bf0a1615446a0dca as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/20a07468e11e4ef8bf0a1615446a0dca 2024-11-27T21:34:39,412 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebbbf218e8d25371b122092896fed576/B of ebbbf218e8d25371b122092896fed576 into 20a07468e11e4ef8bf0a1615446a0dca(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:34:39,412 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:39,412 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., storeName=ebbbf218e8d25371b122092896fed576/B, priority=13, startTime=1732743279281; duration=0sec 2024-11-27T21:34:39,412 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:34:39,412 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebbbf218e8d25371b122092896fed576:B 2024-11-27T21:34:39,415 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebbbf218e8d25371b122092896fed576#C#compaction#57 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:34:39,416 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/3785734deb9b49d3b20eddf7d66f0cf3 is 50, key is test_row_0/C:col10/1732743278482/Put/seqid=0 2024-11-27T21:34:39,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741893_1069 (size=12301) 2024-11-27T21:34:39,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741894_1070 (size=12831) 2024-11-27T21:34:39,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:39,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743339434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:39,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:39,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:39,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:39,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:39,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743339443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:39,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:39,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:39,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:39,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:39,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743339453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:39,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:39,455 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:39,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743339453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:39,455 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:39,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:39,455 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:39,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743339453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:39,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:39,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:39,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:39,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:39,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:39,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:39,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:39,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743339643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:39,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:39,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:39,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:39,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:39,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743339653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:39,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:39,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:39,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:39,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:39,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743339664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:39,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:39,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:39,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:39,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:39,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743339664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:39,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:39,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:39,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:39,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:39,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743339664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:39,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:39,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:39,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:39,820 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/3516d22fcb5d4e758a517537d1d3e934 2024-11-27T21:34:39,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-27T21:34:39,834 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:34:39,835 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:34:39,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-27T21:34:39,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-27T21:34:39,841 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/3785734deb9b49d3b20eddf7d66f0cf3 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/3785734deb9b49d3b20eddf7d66f0cf3 2024-11-27T21:34:39,842 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:34:39,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/0b9d68778ad248f79a6710a040d8ebb1 is 50, key is test_row_0/C:col10/1732743279289/Put/seqid=0 2024-11-27T21:34:39,843 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:34:39,843 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:34:39,860 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ebbbf218e8d25371b122092896fed576/C of ebbbf218e8d25371b122092896fed576 into 3785734deb9b49d3b20eddf7d66f0cf3(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:34:39,860 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:39,860 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., storeName=ebbbf218e8d25371b122092896fed576/C, priority=12, startTime=1732743279282; duration=0sec 2024-11-27T21:34:39,861 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:34:39,861 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebbbf218e8d25371b122092896fed576:C 2024-11-27T21:34:39,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741895_1071 (size=12301) 2024-11-27T21:34:39,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/0b9d68778ad248f79a6710a040d8ebb1 2024-11-27T21:34:39,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/1e4f3bccfe0e48baba3f049078627c37 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/1e4f3bccfe0e48baba3f049078627c37 2024-11-27T21:34:39,891 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/1e4f3bccfe0e48baba3f049078627c37, entries=200, sequenceid=279, filesize=14.4 K 2024-11-27T21:34:39,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/3516d22fcb5d4e758a517537d1d3e934 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/3516d22fcb5d4e758a517537d1d3e934 2024-11-27T21:34:39,898 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/3516d22fcb5d4e758a517537d1d3e934, entries=150, sequenceid=279, filesize=12.0 K 2024-11-27T21:34:39,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/0b9d68778ad248f79a6710a040d8ebb1 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/0b9d68778ad248f79a6710a040d8ebb1 2024-11-27T21:34:39,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/0b9d68778ad248f79a6710a040d8ebb1, entries=150, sequenceid=279, filesize=12.0 K 2024-11-27T21:34:39,909 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for ebbbf218e8d25371b122092896fed576 in 619ms, sequenceid=279, compaction requested=false 2024-11-27T21:34:39,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:39,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-27T21:34:39,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on ebbbf218e8d25371b122092896fed576 2024-11-27T21:34:39,956 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:34:39,956 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ebbbf218e8d25371b122092896fed576 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-27T21:34:39,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/9cc69270007944f6a5fe6507b3681787 is 50, key is test_row_0/A:col10/1732743279328/Put/seqid=0 2024-11-27T21:34:39,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741896_1072 (size=12301) 2024-11-27T21:34:39,980 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/9cc69270007944f6a5fe6507b3681787 2024-11-27T21:34:39,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/0cd3135f51c542f3bb7f54a33d80a2a0 is 50, key is test_row_0/B:col10/1732743279328/Put/seqid=0 2024-11-27T21:34:40,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741897_1073 (size=12301) 2024-11-27T21:34:40,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:40,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743339994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:40,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/0cd3135f51c542f3bb7f54a33d80a2a0 2024-11-27T21:34:40,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:40,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743339994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:40,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-27T21:34:40,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:40,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:40,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:40,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:40,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:40,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743339995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:40,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:40,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:40,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:40,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:40,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743339996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:40,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:40,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743339997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:40,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:40,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:40,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:40,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:40,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/3c3b35bc105b4cafbf1c44f94ea8908e is 50, key is test_row_0/C:col10/1732743279328/Put/seqid=0 2024-11-27T21:34:40,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741898_1074 (size=12301) 2024-11-27T21:34:40,026 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/3c3b35bc105b4cafbf1c44f94ea8908e 2024-11-27T21:34:40,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/9cc69270007944f6a5fe6507b3681787 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/9cc69270007944f6a5fe6507b3681787 2024-11-27T21:34:40,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/9cc69270007944f6a5fe6507b3681787, entries=150, sequenceid=300, filesize=12.0 K 2024-11-27T21:34:40,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/0cd3135f51c542f3bb7f54a33d80a2a0 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/0cd3135f51c542f3bb7f54a33d80a2a0 2024-11-27T21:34:40,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/0cd3135f51c542f3bb7f54a33d80a2a0, entries=150, sequenceid=300, filesize=12.0 K 2024-11-27T21:34:40,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/3c3b35bc105b4cafbf1c44f94ea8908e as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/3c3b35bc105b4cafbf1c44f94ea8908e 2024-11-27T21:34:40,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/3c3b35bc105b4cafbf1c44f94ea8908e, entries=150, sequenceid=300, filesize=12.0 K 2024-11-27T21:34:40,057 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for ebbbf218e8d25371b122092896fed576 in 101ms, sequenceid=300, compaction requested=true 2024-11-27T21:34:40,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:40,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebbbf218e8d25371b122092896fed576:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:34:40,057 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:34:40,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:34:40,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebbbf218e8d25371b122092896fed576:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:34:40,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:34:40,057 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:34:40,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebbbf218e8d25371b122092896fed576:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:34:40,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:34:40,058 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39907 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:34:40,058 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): ebbbf218e8d25371b122092896fed576/A is initiating minor compaction (all files) 2024-11-27T21:34:40,058 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebbbf218e8d25371b122092896fed576/A in TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:40,059 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/51dfcc094e9a4363ad69d5e429c4e333, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/1e4f3bccfe0e48baba3f049078627c37, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/9cc69270007944f6a5fe6507b3681787] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp, totalSize=39.0 K 2024-11-27T21:34:40,059 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37467 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:34:40,059 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): ebbbf218e8d25371b122092896fed576/B is initiating minor compaction (all files) 2024-11-27T21:34:40,059 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebbbf218e8d25371b122092896fed576/B in TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:40,059 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/20a07468e11e4ef8bf0a1615446a0dca, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/3516d22fcb5d4e758a517537d1d3e934, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/0cd3135f51c542f3bb7f54a33d80a2a0] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp, totalSize=36.6 K 2024-11-27T21:34:40,059 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51dfcc094e9a4363ad69d5e429c4e333, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732743278482 2024-11-27T21:34:40,060 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e4f3bccfe0e48baba3f049078627c37, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732743279168 2024-11-27T21:34:40,060 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 20a07468e11e4ef8bf0a1615446a0dca, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732743278482 2024-11-27T21:34:40,060 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 3516d22fcb5d4e758a517537d1d3e934, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732743279170 2024-11-27T21:34:40,060 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cc69270007944f6a5fe6507b3681787, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1732743279300 2024-11-27T21:34:40,061 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cd3135f51c542f3bb7f54a33d80a2a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1732743279300 2024-11-27T21:34:40,070 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebbbf218e8d25371b122092896fed576#A#compaction#62 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:34:40,071 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebbbf218e8d25371b122092896fed576#B#compaction#63 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:34:40,071 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/ba6aa291d9e5420f95bbf773692e0652 is 50, key is test_row_0/A:col10/1732743279328/Put/seqid=0 2024-11-27T21:34:40,071 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/62fa5f33b32b41e393b66d86ba261376 is 50, key is test_row_0/B:col10/1732743279328/Put/seqid=0 2024-11-27T21:34:40,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741899_1075 (size=13017) 2024-11-27T21:34:40,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741900_1076 (size=13017) 2024-11-27T21:34:40,088 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/ba6aa291d9e5420f95bbf773692e0652 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/ba6aa291d9e5420f95bbf773692e0652 2024-11-27T21:34:40,095 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebbbf218e8d25371b122092896fed576/A of ebbbf218e8d25371b122092896fed576 into ba6aa291d9e5420f95bbf773692e0652(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:34:40,095 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:40,095 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., storeName=ebbbf218e8d25371b122092896fed576/A, priority=13, startTime=1732743280057; duration=0sec 2024-11-27T21:34:40,096 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:34:40,096 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebbbf218e8d25371b122092896fed576:A 2024-11-27T21:34:40,096 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:34:40,097 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37433 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:34:40,097 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): ebbbf218e8d25371b122092896fed576/C is initiating minor compaction (all files) 2024-11-27T21:34:40,097 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebbbf218e8d25371b122092896fed576/C in TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:40,097 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/3785734deb9b49d3b20eddf7d66f0cf3, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/0b9d68778ad248f79a6710a040d8ebb1, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/3c3b35bc105b4cafbf1c44f94ea8908e] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp, totalSize=36.6 K 2024-11-27T21:34:40,098 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3785734deb9b49d3b20eddf7d66f0cf3, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732743278482 2024-11-27T21:34:40,098 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b9d68778ad248f79a6710a040d8ebb1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732743279170 2024-11-27T21:34:40,099 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c3b35bc105b4cafbf1c44f94ea8908e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1732743279300 2024-11-27T21:34:40,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on ebbbf218e8d25371b122092896fed576 2024-11-27T21:34:40,107 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:34:40,107 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ebbbf218e8d25371b122092896fed576 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-27T21:34:40,113 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebbbf218e8d25371b122092896fed576#C#compaction#64 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:34:40,114 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/6a466c7264004a3cbf8ed003e28dfa64 is 50, key is test_row_0/C:col10/1732743279328/Put/seqid=0 2024-11-27T21:34:40,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/28c40e72b0f9438d8c8e0392a93aa99c is 50, key is test_row_0/A:col10/1732743280105/Put/seqid=0 2024-11-27T21:34:40,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:40,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743340137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:40,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:40,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743340138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:40,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:40,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:40,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:40,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743340140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:40,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:40,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743340139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:40,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:40,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:40,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:40,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743340141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:40,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:40,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741901_1077 (size=12983) 2024-11-27T21:34:40,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-27T21:34:40,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-27T21:34:40,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:40,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:40,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:40,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:40,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:40,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:40,160 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/6a466c7264004a3cbf8ed003e28dfa64 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/6a466c7264004a3cbf8ed003e28dfa64 2024-11-27T21:34:40,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741902_1078 (size=12301) 2024-11-27T21:34:40,171 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebbbf218e8d25371b122092896fed576/C of ebbbf218e8d25371b122092896fed576 into 6a466c7264004a3cbf8ed003e28dfa64(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:34:40,171 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:40,171 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., storeName=ebbbf218e8d25371b122092896fed576/C, priority=13, startTime=1732743280057; duration=0sec 2024-11-27T21:34:40,171 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:34:40,171 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebbbf218e8d25371b122092896fed576:C 2024-11-27T21:34:40,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:40,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743340254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:40,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:40,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743340254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:40,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:40,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:40,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743340254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:40,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:40,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743340254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:40,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:40,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:40,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:40,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:40,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743340255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:40,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:40,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-27T21:34:40,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:40,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:40,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:40,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:40,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:40,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:40,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-27T21:34:40,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-27T21:34:40,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:40,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:40,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:40,466 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:40,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:40,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:40,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743340463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:40,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:40,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:40,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:40,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743340464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:40,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:40,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:40,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:40,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743340464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:40,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743340464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:40,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:40,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743340464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:40,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:40,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:40,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:40,490 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/62fa5f33b32b41e393b66d86ba261376 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/62fa5f33b32b41e393b66d86ba261376 2024-11-27T21:34:40,498 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebbbf218e8d25371b122092896fed576/B of ebbbf218e8d25371b122092896fed576 into 62fa5f33b32b41e393b66d86ba261376(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:34:40,498 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:40,498 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., storeName=ebbbf218e8d25371b122092896fed576/B, priority=13, startTime=1732743280057; duration=0sec 2024-11-27T21:34:40,498 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:34:40,498 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebbbf218e8d25371b122092896fed576:B 2024-11-27T21:34:40,570 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/28c40e72b0f9438d8c8e0392a93aa99c 2024-11-27T21:34:40,581 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/a8b9a39bdeff4d96888389dcb2e2c3ec is 50, key is test_row_0/B:col10/1732743280105/Put/seqid=0 2024-11-27T21:34:40,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741903_1079 (size=12301) 2024-11-27T21:34:40,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-27T21:34:40,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:40,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:40,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:40,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:40,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:40,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:40,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-27T21:34:40,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:40,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:40,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:40,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:40,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:40,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:40,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:40,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743340774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:40,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:40,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:40,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743340774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:40,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:40,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:40,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743340774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:40,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:40,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:40,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743340783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:40,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:40,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743340783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:40,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:40,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:40,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:40,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-27T21:34:40,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:40,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:40,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:40,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:40,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:40,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:40,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-27T21:34:40,989 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/a8b9a39bdeff4d96888389dcb2e2c3ec 2024-11-27T21:34:40,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/0ea9fab9cd654ba798b6b3f89e0fa719 is 50, key is test_row_0/C:col10/1732743280105/Put/seqid=0 2024-11-27T21:34:41,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741904_1080 (size=12301) 2024-11-27T21:34:41,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-27T21:34:41,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:41,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:41,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:41,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:41,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:41,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:41,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-27T21:34:41,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:41,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:41,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:41,237 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:41,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:41,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:41,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:41,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743341284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:41,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:41,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:41,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:41,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:41,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743341284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:41,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:41,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:41,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:41,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:41,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743341284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:41,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:41,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:41,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:41,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:41,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743341294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:41,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:41,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743341294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:41,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:41,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:41,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:41,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:41,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:41,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:41,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-27T21:34:41,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:41,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:41,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:41,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:41,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:41,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:41,406 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/0ea9fab9cd654ba798b6b3f89e0fa719 2024-11-27T21:34:41,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/28c40e72b0f9438d8c8e0392a93aa99c as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/28c40e72b0f9438d8c8e0392a93aa99c 2024-11-27T21:34:41,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/28c40e72b0f9438d8c8e0392a93aa99c, entries=150, sequenceid=320, filesize=12.0 K 2024-11-27T21:34:41,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/a8b9a39bdeff4d96888389dcb2e2c3ec as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/a8b9a39bdeff4d96888389dcb2e2c3ec 2024-11-27T21:34:41,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/a8b9a39bdeff4d96888389dcb2e2c3ec, entries=150, sequenceid=320, filesize=12.0 K 2024-11-27T21:34:41,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/0ea9fab9cd654ba798b6b3f89e0fa719 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/0ea9fab9cd654ba798b6b3f89e0fa719 2024-11-27T21:34:41,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/0ea9fab9cd654ba798b6b3f89e0fa719, entries=150, sequenceid=320, filesize=12.0 K 2024-11-27T21:34:41,438 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for ebbbf218e8d25371b122092896fed576 in 1331ms, sequenceid=320, compaction requested=false 2024-11-27T21:34:41,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:41,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-27T21:34:41,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:41,548 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing ebbbf218e8d25371b122092896fed576 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-27T21:34:41,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/89a1482c64d443978b1e850ca7a7e537 is 50, key is test_row_0/A:col10/1732743280139/Put/seqid=0 2024-11-27T21:34:41,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741905_1081 (size=9857) 2024-11-27T21:34:41,980 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/89a1482c64d443978b1e850ca7a7e537 2024-11-27T21:34:41,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-27T21:34:41,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/b35e86ca12a842008367651522ef63ff is 50, key is test_row_0/B:col10/1732743280139/Put/seqid=0 2024-11-27T21:34:42,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741906_1082 (size=9857) 2024-11-27T21:34:42,003 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/b35e86ca12a842008367651522ef63ff 2024-11-27T21:34:42,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/c4afcc83fdeb491a8acaf1335a200e2d is 50, key is test_row_0/C:col10/1732743280139/Put/seqid=0 2024-11-27T21:34:42,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741907_1083 (size=9857) 2024-11-27T21:34:42,031 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/c4afcc83fdeb491a8acaf1335a200e2d 2024-11-27T21:34:42,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/89a1482c64d443978b1e850ca7a7e537 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/89a1482c64d443978b1e850ca7a7e537 2024-11-27T21:34:42,053 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/89a1482c64d443978b1e850ca7a7e537, entries=100, sequenceid=339, filesize=9.6 K 2024-11-27T21:34:42,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/b35e86ca12a842008367651522ef63ff as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/b35e86ca12a842008367651522ef63ff 2024-11-27T21:34:42,067 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/b35e86ca12a842008367651522ef63ff, entries=100, sequenceid=339, filesize=9.6 K 2024-11-27T21:34:42,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/c4afcc83fdeb491a8acaf1335a200e2d as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/c4afcc83fdeb491a8acaf1335a200e2d 2024-11-27T21:34:42,079 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/c4afcc83fdeb491a8acaf1335a200e2d, entries=100, sequenceid=339, filesize=9.6 K 2024-11-27T21:34:42,081 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=0 B/0 for ebbbf218e8d25371b122092896fed576 in 533ms, sequenceid=339, compaction requested=true 2024-11-27T21:34:42,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:42,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:42,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-27T21:34:42,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-27T21:34:42,087 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-27T21:34:42,087 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2400 sec 2024-11-27T21:34:42,092 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 2.2530 sec 2024-11-27T21:34:42,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on ebbbf218e8d25371b122092896fed576 2024-11-27T21:34:42,322 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:34:42,322 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ebbbf218e8d25371b122092896fed576 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T21:34:42,333 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/bf8f6c2f554f4043bda9c83d4336e03f is 50, key is test_row_0/A:col10/1732743282319/Put/seqid=0 2024-11-27T21:34:42,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741908_1084 (size=12301) 2024-11-27T21:34:42,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:42,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743342389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:42,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:42,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:42,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:42,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:42,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743342391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:42,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:42,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:42,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:42,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:42,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743342393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:42,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:42,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:42,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:42,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:42,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:42,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743342393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:42,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743342393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:42,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:42,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:42,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:42,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:42,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:42,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:42,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:42,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743342503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:42,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:42,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:42,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:42,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:42,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743342503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:42,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:42,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:42,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:42,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:42,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743342513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:42,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:42,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743342514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:42,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:42,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:42,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:42,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:42,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743342514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:42,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:42,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:42,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:42,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:42,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:42,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:42,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:42,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743342713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:42,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:42,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:42,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:42,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:42,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743342714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:42,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:42,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:42,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:42,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:42,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743342723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:42,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:42,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:42,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:42,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:42,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743342723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:42,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:42,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743342723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:42,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:42,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:42,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:42,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:42,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:42,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:42,751 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/bf8f6c2f554f4043bda9c83d4336e03f 2024-11-27T21:34:42,760 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/90bea62db87847acaf2635dba524f8c2 is 50, key is test_row_0/B:col10/1732743282319/Put/seqid=0 2024-11-27T21:34:42,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741909_1085 (size=12301) 2024-11-27T21:34:42,765 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/90bea62db87847acaf2635dba524f8c2 2024-11-27T21:34:42,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/fe9a296d8fc443c8a259feadc02391f8 is 50, key is test_row_0/C:col10/1732743282319/Put/seqid=0 2024-11-27T21:34:42,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741910_1086 (size=12301) 2024-11-27T21:34:43,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:43,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743343023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:43,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:43,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:43,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:43,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:43,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743343023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:43,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:43,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:43,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:43,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:43,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743343033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:43,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:43,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:43,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:43,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:43,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743343034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:43,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:43,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:43,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:43,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:43,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743343034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:43,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:43,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:43,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:43,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/fe9a296d8fc443c8a259feadc02391f8 2024-11-27T21:34:43,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/bf8f6c2f554f4043bda9c83d4336e03f as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/bf8f6c2f554f4043bda9c83d4336e03f 2024-11-27T21:34:43,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/bf8f6c2f554f4043bda9c83d4336e03f, entries=150, sequenceid=351, filesize=12.0 K 2024-11-27T21:34:43,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/90bea62db87847acaf2635dba524f8c2 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/90bea62db87847acaf2635dba524f8c2 2024-11-27T21:34:43,206 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/90bea62db87847acaf2635dba524f8c2, entries=150, sequenceid=351, filesize=12.0 K 2024-11-27T21:34:43,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/fe9a296d8fc443c8a259feadc02391f8 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/fe9a296d8fc443c8a259feadc02391f8 2024-11-27T21:34:43,212 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/fe9a296d8fc443c8a259feadc02391f8, entries=150, sequenceid=351, filesize=12.0 K 2024-11-27T21:34:43,213 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for ebbbf218e8d25371b122092896fed576 in 891ms, sequenceid=351, compaction requested=true 2024-11-27T21:34:43,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:43,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebbbf218e8d25371b122092896fed576:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:34:43,213 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:34:43,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:34:43,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebbbf218e8d25371b122092896fed576:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:34:43,214 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:34:43,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:34:43,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebbbf218e8d25371b122092896fed576:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:34:43,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:34:43,215 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47476 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:34:43,215 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47476 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:34:43,215 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): ebbbf218e8d25371b122092896fed576/B is initiating minor compaction (all files) 2024-11-27T21:34:43,215 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): ebbbf218e8d25371b122092896fed576/A is initiating minor compaction (all files) 2024-11-27T21:34:43,215 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebbbf218e8d25371b122092896fed576/B in TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:43,215 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebbbf218e8d25371b122092896fed576/A in TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:43,215 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/62fa5f33b32b41e393b66d86ba261376, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/a8b9a39bdeff4d96888389dcb2e2c3ec, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/b35e86ca12a842008367651522ef63ff, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/90bea62db87847acaf2635dba524f8c2] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp, totalSize=46.4 K 2024-11-27T21:34:43,215 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/ba6aa291d9e5420f95bbf773692e0652, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/28c40e72b0f9438d8c8e0392a93aa99c, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/89a1482c64d443978b1e850ca7a7e537, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/bf8f6c2f554f4043bda9c83d4336e03f] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp, totalSize=46.4 K 2024-11-27T21:34:43,216 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 62fa5f33b32b41e393b66d86ba261376, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1732743279300 2024-11-27T21:34:43,220 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba6aa291d9e5420f95bbf773692e0652, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1732743279300 2024-11-27T21:34:43,220 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting a8b9a39bdeff4d96888389dcb2e2c3ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732743279992 2024-11-27T21:34:43,221 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28c40e72b0f9438d8c8e0392a93aa99c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732743279992 2024-11-27T21:34:43,221 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting b35e86ca12a842008367651522ef63ff, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732743280136 2024-11-27T21:34:43,221 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89a1482c64d443978b1e850ca7a7e537, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732743280136 2024-11-27T21:34:43,221 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 90bea62db87847acaf2635dba524f8c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1732743282319 2024-11-27T21:34:43,222 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf8f6c2f554f4043bda9c83d4336e03f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1732743282319 2024-11-27T21:34:43,234 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebbbf218e8d25371b122092896fed576#B#compaction#74 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:34:43,234 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebbbf218e8d25371b122092896fed576#A#compaction#75 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:34:43,235 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/287c9052832a43c09459c5b178da4fa5 is 50, key is test_row_0/A:col10/1732743282319/Put/seqid=0 2024-11-27T21:34:43,236 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/08bf262bd3354cbab381cebebb9383e8 is 50, key is test_row_0/B:col10/1732743282319/Put/seqid=0 2024-11-27T21:34:43,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741912_1088 (size=13153) 2024-11-27T21:34:43,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741911_1087 (size=13153) 2024-11-27T21:34:43,251 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/08bf262bd3354cbab381cebebb9383e8 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/08bf262bd3354cbab381cebebb9383e8 2024-11-27T21:34:43,258 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ebbbf218e8d25371b122092896fed576/B of ebbbf218e8d25371b122092896fed576 into 08bf262bd3354cbab381cebebb9383e8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:34:43,258 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:43,258 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., storeName=ebbbf218e8d25371b122092896fed576/B, priority=12, startTime=1732743283214; duration=0sec 2024-11-27T21:34:43,258 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:34:43,258 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebbbf218e8d25371b122092896fed576:B 2024-11-27T21:34:43,258 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:34:43,261 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47442 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:34:43,261 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): ebbbf218e8d25371b122092896fed576/C is initiating minor compaction (all files) 2024-11-27T21:34:43,261 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebbbf218e8d25371b122092896fed576/C in TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:43,261 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/6a466c7264004a3cbf8ed003e28dfa64, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/0ea9fab9cd654ba798b6b3f89e0fa719, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/c4afcc83fdeb491a8acaf1335a200e2d, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/fe9a296d8fc443c8a259feadc02391f8] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp, totalSize=46.3 K 2024-11-27T21:34:43,262 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a466c7264004a3cbf8ed003e28dfa64, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1732743279300 2024-11-27T21:34:43,262 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ea9fab9cd654ba798b6b3f89e0fa719, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732743279992 2024-11-27T21:34:43,263 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting c4afcc83fdeb491a8acaf1335a200e2d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732743280136 2024-11-27T21:34:43,263 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting fe9a296d8fc443c8a259feadc02391f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1732743282319 2024-11-27T21:34:43,275 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebbbf218e8d25371b122092896fed576#C#compaction#76 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:34:43,276 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/e28cd0752e894c0a8bfbaceba0e4aead is 50, key is test_row_0/C:col10/1732743282319/Put/seqid=0 2024-11-27T21:34:43,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741913_1089 (size=13119) 2024-11-27T21:34:43,547 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:34:43,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on ebbbf218e8d25371b122092896fed576 2024-11-27T21:34:43,548 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ebbbf218e8d25371b122092896fed576 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-27T21:34:43,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/1e13b19d982a4407ba498b874b0929db is 50, key is test_row_0/A:col10/1732743283545/Put/seqid=0 2024-11-27T21:34:43,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:43,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743343553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:43,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:43,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:43,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:43,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:43,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743343557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:43,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:43,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743343558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:43,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:43,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743343559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:43,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:43,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:43,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:43,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:43,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:43,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743343559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:43,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:43,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:43,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:43,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:43,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:43,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:43,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:43,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:43,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741914_1090 (size=17181) 2024-11-27T21:34:43,578 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/1e13b19d982a4407ba498b874b0929db 2024-11-27T21:34:43,589 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/4d807145557c41aa88020532881f6e0c is 50, key is test_row_0/B:col10/1732743283545/Put/seqid=0 2024-11-27T21:34:43,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741915_1091 (size=12301) 2024-11-27T21:34:43,598 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/4d807145557c41aa88020532881f6e0c 2024-11-27T21:34:43,612 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/691aa8864c8c40ed8ff7c9f416077b67 is 50, key is test_row_0/C:col10/1732743283545/Put/seqid=0 2024-11-27T21:34:43,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741916_1092 (size=12301) 2024-11-27T21:34:43,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/691aa8864c8c40ed8ff7c9f416077b67 2024-11-27T21:34:43,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/1e13b19d982a4407ba498b874b0929db as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/1e13b19d982a4407ba498b874b0929db 2024-11-27T21:34:43,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/1e13b19d982a4407ba498b874b0929db, entries=250, sequenceid=378, filesize=16.8 K 2024-11-27T21:34:43,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/4d807145557c41aa88020532881f6e0c as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/4d807145557c41aa88020532881f6e0c 2024-11-27T21:34:43,654 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/287c9052832a43c09459c5b178da4fa5 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/287c9052832a43c09459c5b178da4fa5 2024-11-27T21:34:43,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/4d807145557c41aa88020532881f6e0c, entries=150, sequenceid=378, filesize=12.0 K 2024-11-27T21:34:43,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/691aa8864c8c40ed8ff7c9f416077b67 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/691aa8864c8c40ed8ff7c9f416077b67 2024-11-27T21:34:43,664 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ebbbf218e8d25371b122092896fed576/A of ebbbf218e8d25371b122092896fed576 into 287c9052832a43c09459c5b178da4fa5(size=12.8 K), total size for store is 29.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:34:43,664 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:43,665 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., storeName=ebbbf218e8d25371b122092896fed576/A, priority=12, startTime=1732743283213; duration=0sec 2024-11-27T21:34:43,666 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:34:43,666 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebbbf218e8d25371b122092896fed576:A 2024-11-27T21:34:43,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:43,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743343664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:43,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:43,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:43,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:43,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/691aa8864c8c40ed8ff7c9f416077b67, entries=150, sequenceid=378, filesize=12.0 K 2024-11-27T21:34:43,675 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for ebbbf218e8d25371b122092896fed576 in 128ms, sequenceid=378, compaction requested=false 2024-11-27T21:34:43,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:43,684 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:34:43,684 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ebbbf218e8d25371b122092896fed576 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T21:34:43,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on ebbbf218e8d25371b122092896fed576 2024-11-27T21:34:43,692 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/e28cd0752e894c0a8bfbaceba0e4aead as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/e28cd0752e894c0a8bfbaceba0e4aead 2024-11-27T21:34:43,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/192caad9dd5a4bfe8a9bd9868826569c is 50, key is test_row_0/A:col10/1732743283557/Put/seqid=0 2024-11-27T21:34:43,704 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ebbbf218e8d25371b122092896fed576/C of ebbbf218e8d25371b122092896fed576 into e28cd0752e894c0a8bfbaceba0e4aead(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:34:43,704 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:43,704 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., storeName=ebbbf218e8d25371b122092896fed576/C, priority=12, startTime=1732743283214; duration=0sec 2024-11-27T21:34:43,705 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:34:43,705 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebbbf218e8d25371b122092896fed576:C 2024-11-27T21:34:43,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741917_1093 (size=12301) 2024-11-27T21:34:43,716 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/192caad9dd5a4bfe8a9bd9868826569c 2024-11-27T21:34:43,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/28d9674c034041dfb58408ecad093124 is 50, key is test_row_0/B:col10/1732743283557/Put/seqid=0 2024-11-27T21:34:43,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:43,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:43,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741918_1094 (size=12301) 2024-11-27T21:34:43,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743343735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:43,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743343736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:43,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:43,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743343744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:43,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:43,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:43,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:43,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:43,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743343745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:43,753 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/28d9674c034041dfb58408ecad093124 2024-11-27T21:34:43,772 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/51c8051f8781422fa3f3e7fa8d4db854 is 50, key is test_row_0/C:col10/1732743283557/Put/seqid=0 2024-11-27T21:34:43,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741919_1095 (size=12301) 2024-11-27T21:34:43,784 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/51c8051f8781422fa3f3e7fa8d4db854 2024-11-27T21:34:43,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/192caad9dd5a4bfe8a9bd9868826569c as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/192caad9dd5a4bfe8a9bd9868826569c 2024-11-27T21:34:43,808 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/192caad9dd5a4bfe8a9bd9868826569c, entries=150, sequenceid=391, filesize=12.0 K 2024-11-27T21:34:43,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/28d9674c034041dfb58408ecad093124 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/28d9674c034041dfb58408ecad093124 2024-11-27T21:34:43,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/28d9674c034041dfb58408ecad093124, entries=150, sequenceid=391, filesize=12.0 K 2024-11-27T21:34:43,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/51c8051f8781422fa3f3e7fa8d4db854 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/51c8051f8781422fa3f3e7fa8d4db854 2024-11-27T21:34:43,825 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/51c8051f8781422fa3f3e7fa8d4db854, entries=150, sequenceid=391, filesize=12.0 K 2024-11-27T21:34:43,826 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for ebbbf218e8d25371b122092896fed576 in 142ms, sequenceid=391, compaction requested=true 2024-11-27T21:34:43,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:43,827 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:34:43,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebbbf218e8d25371b122092896fed576:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:34:43,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:34:43,827 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:34:43,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebbbf218e8d25371b122092896fed576:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:34:43,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:34:43,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebbbf218e8d25371b122092896fed576:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:34:43,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:34:43,828 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42635 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:34:43,828 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): ebbbf218e8d25371b122092896fed576/A is initiating minor compaction (all files) 2024-11-27T21:34:43,828 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebbbf218e8d25371b122092896fed576/A in TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:43,828 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:34:43,828 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/287c9052832a43c09459c5b178da4fa5, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/1e13b19d982a4407ba498b874b0929db, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/192caad9dd5a4bfe8a9bd9868826569c] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp, totalSize=41.6 K 2024-11-27T21:34:43,828 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): ebbbf218e8d25371b122092896fed576/B is initiating minor compaction (all files) 2024-11-27T21:34:43,828 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebbbf218e8d25371b122092896fed576/B in TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:43,829 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/08bf262bd3354cbab381cebebb9383e8, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/4d807145557c41aa88020532881f6e0c, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/28d9674c034041dfb58408ecad093124] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp, totalSize=36.9 K 2024-11-27T21:34:43,829 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 287c9052832a43c09459c5b178da4fa5, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1732743282319 2024-11-27T21:34:43,829 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 08bf262bd3354cbab381cebebb9383e8, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1732743282319 2024-11-27T21:34:43,830 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d807145557c41aa88020532881f6e0c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732743282387 2024-11-27T21:34:43,830 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e13b19d982a4407ba498b874b0929db, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732743282387 2024-11-27T21:34:43,830 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 28d9674c034041dfb58408ecad093124, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1732743283557 2024-11-27T21:34:43,830 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 192caad9dd5a4bfe8a9bd9868826569c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1732743283557 2024-11-27T21:34:43,842 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebbbf218e8d25371b122092896fed576#A#compaction#83 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:34:43,843 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebbbf218e8d25371b122092896fed576#B#compaction#84 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:34:43,843 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/b645eec5b9fb46ddbe7f396a7b3ad4ec is 50, key is test_row_0/A:col10/1732743283557/Put/seqid=0 2024-11-27T21:34:43,843 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/21badf024c25408cb484ee62eaa61569 is 50, key is test_row_0/B:col10/1732743283557/Put/seqid=0 2024-11-27T21:34:43,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on ebbbf218e8d25371b122092896fed576 2024-11-27T21:34:43,867 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:34:43,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741920_1096 (size=13255) 2024-11-27T21:34:43,868 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ebbbf218e8d25371b122092896fed576 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-27T21:34:43,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741921_1097 (size=13255) 2024-11-27T21:34:43,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/bade09d1cec64c30ab8ed239c09661ed is 50, key is test_row_0/A:col10/1732743283862/Put/seqid=0 2024-11-27T21:34:43,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:43,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743343880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:43,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:43,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:43,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743343881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:43,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743343880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:43,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:43,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:43,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:43,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743343883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:43,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:43,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:43,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:43,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743343883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:43,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:43,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:43,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:43,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:43,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:43,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:43,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:43,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:43,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:43,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:43,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:43,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741922_1098 (size=12301) 2024-11-27T21:34:43,895 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/bade09d1cec64c30ab8ed239c09661ed 2024-11-27T21:34:43,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/d667b1fbf6b34c92a61749030ec94121 is 50, key is test_row_0/B:col10/1732743283862/Put/seqid=0 2024-11-27T21:34:43,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741923_1099 (size=12301) 2024-11-27T21:34:44,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:44,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743343993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:44,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:44,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:44,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743343993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:44,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:44,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743343993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:44,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:44,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743343994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:44,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:44,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:44,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743343994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:44,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:44,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:44,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-27T21:34:44,004 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:34:44,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:44,005 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:34:44,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-27T21:34:44,008 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:34:44,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-27T21:34:44,009 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:34:44,010 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:34:44,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-27T21:34:44,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-27T21:34:44,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:44,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:44,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:44,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:44,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:44,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:44,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:44,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743344204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:44,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:44,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:44,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743344213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:44,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:44,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743344213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:44,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:44,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743344213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:44,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:44,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:44,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:44,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743344213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:44,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:44,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:44,278 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/21badf024c25408cb484ee62eaa61569 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/21badf024c25408cb484ee62eaa61569 2024-11-27T21:34:44,278 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/b645eec5b9fb46ddbe7f396a7b3ad4ec as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/b645eec5b9fb46ddbe7f396a7b3ad4ec 2024-11-27T21:34:44,286 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebbbf218e8d25371b122092896fed576/B of ebbbf218e8d25371b122092896fed576 into 21badf024c25408cb484ee62eaa61569(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:34:44,286 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:44,287 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., storeName=ebbbf218e8d25371b122092896fed576/B, priority=13, startTime=1732743283827; duration=0sec 2024-11-27T21:34:44,287 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:34:44,287 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebbbf218e8d25371b122092896fed576:B 2024-11-27T21:34:44,287 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:34:44,288 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebbbf218e8d25371b122092896fed576/A of ebbbf218e8d25371b122092896fed576 into b645eec5b9fb46ddbe7f396a7b3ad4ec(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:34:44,288 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:44,288 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., storeName=ebbbf218e8d25371b122092896fed576/A, priority=13, startTime=1732743283826; duration=0sec 2024-11-27T21:34:44,288 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:34:44,288 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebbbf218e8d25371b122092896fed576:A 2024-11-27T21:34:44,289 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:34:44,289 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): ebbbf218e8d25371b122092896fed576/C is initiating minor compaction (all files) 2024-11-27T21:34:44,289 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebbbf218e8d25371b122092896fed576/C in TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:44,289 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/e28cd0752e894c0a8bfbaceba0e4aead, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/691aa8864c8c40ed8ff7c9f416077b67, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/51c8051f8781422fa3f3e7fa8d4db854] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp, totalSize=36.8 K 2024-11-27T21:34:44,290 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting e28cd0752e894c0a8bfbaceba0e4aead, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1732743282319 2024-11-27T21:34:44,290 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 691aa8864c8c40ed8ff7c9f416077b67, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732743282387 2024-11-27T21:34:44,291 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 51c8051f8781422fa3f3e7fa8d4db854, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1732743283557 2024-11-27T21:34:44,302 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebbbf218e8d25371b122092896fed576#C#compaction#87 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:34:44,303 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/2b38bcf7829f4bd6a1dbee4b1408135a is 50, key is test_row_0/C:col10/1732743283557/Put/seqid=0 2024-11-27T21:34:44,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741924_1100 (size=13221) 2024-11-27T21:34:44,316 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/d667b1fbf6b34c92a61749030ec94121 2024-11-27T21:34:44,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-27T21:34:44,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:44,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:44,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:44,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:44,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:44,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:44,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-27T21:34:44,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/974899bfd3294a46ace9eb40c7c58472 is 50, key is test_row_0/C:col10/1732743283862/Put/seqid=0 2024-11-27T21:34:44,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741925_1101 (size=12301) 2024-11-27T21:34:44,339 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/974899bfd3294a46ace9eb40c7c58472 2024-11-27T21:34:44,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/bade09d1cec64c30ab8ed239c09661ed as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/bade09d1cec64c30ab8ed239c09661ed 2024-11-27T21:34:44,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/bade09d1cec64c30ab8ed239c09661ed, entries=150, sequenceid=420, filesize=12.0 K 2024-11-27T21:34:44,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/d667b1fbf6b34c92a61749030ec94121 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/d667b1fbf6b34c92a61749030ec94121 2024-11-27T21:34:44,360 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/d667b1fbf6b34c92a61749030ec94121, entries=150, sequenceid=420, filesize=12.0 K 2024-11-27T21:34:44,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/974899bfd3294a46ace9eb40c7c58472 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/974899bfd3294a46ace9eb40c7c58472 2024-11-27T21:34:44,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/974899bfd3294a46ace9eb40c7c58472, entries=150, sequenceid=420, filesize=12.0 K 2024-11-27T21:34:44,371 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=40.25 KB/41220 for ebbbf218e8d25371b122092896fed576 in 504ms, sequenceid=420, compaction requested=false 2024-11-27T21:34:44,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:44,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-27T21:34:44,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:44,474 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing ebbbf218e8d25371b122092896fed576 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-27T21:34:44,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/0d49f2abcf6d4baab5543e79008a6265 is 50, key is test_row_0/A:col10/1732743283878/Put/seqid=0 2024-11-27T21:34:44,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741926_1102 (size=12301) 2024-11-27T21:34:44,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on ebbbf218e8d25371b122092896fed576 2024-11-27T21:34:44,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:44,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:44,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743344550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:44,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:44,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:44,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743344552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:44,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:44,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:44,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743344554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:44,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:44,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:44,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743344555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:44,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:44,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:44,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743344557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:44,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:44,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-27T21:34:44,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:44,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743344664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:44,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:44,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:44,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743344673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:44,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:44,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:44,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743344673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:44,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:44,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:44,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743344673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:44,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:44,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:44,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743344674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:44,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:44,718 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/2b38bcf7829f4bd6a1dbee4b1408135a as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/2b38bcf7829f4bd6a1dbee4b1408135a 2024-11-27T21:34:44,726 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebbbf218e8d25371b122092896fed576/C of ebbbf218e8d25371b122092896fed576 into 2b38bcf7829f4bd6a1dbee4b1408135a(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:34:44,726 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:44,726 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., storeName=ebbbf218e8d25371b122092896fed576/C, priority=13, startTime=1732743283827; duration=0sec 2024-11-27T21:34:44,726 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:34:44,726 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebbbf218e8d25371b122092896fed576:C 2024-11-27T21:34:44,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:44,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743344873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:44,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:44,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:44,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743344883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:44,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:44,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743344884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:44,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:44,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:44,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743344884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:44,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:44,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:44,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:44,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743344884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:44,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:44,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:44,899 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/0d49f2abcf6d4baab5543e79008a6265 2024-11-27T21:34:44,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/89d3d6af96b2444bbac6d6b75fa62b0f is 50, key is test_row_0/B:col10/1732743283878/Put/seqid=0 2024-11-27T21:34:44,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741927_1103 (size=12301) 2024-11-27T21:34:44,929 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/89d3d6af96b2444bbac6d6b75fa62b0f 2024-11-27T21:34:44,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/60527239776b43c8894658bb85bc2a4a is 50, key is test_row_0/C:col10/1732743283878/Put/seqid=0 2024-11-27T21:34:44,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741928_1104 (size=12301) 2024-11-27T21:34:44,957 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/60527239776b43c8894658bb85bc2a4a 2024-11-27T21:34:44,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/0d49f2abcf6d4baab5543e79008a6265 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/0d49f2abcf6d4baab5543e79008a6265 2024-11-27T21:34:44,973 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/0d49f2abcf6d4baab5543e79008a6265, entries=150, sequenceid=431, filesize=12.0 K 2024-11-27T21:34:44,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/89d3d6af96b2444bbac6d6b75fa62b0f as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/89d3d6af96b2444bbac6d6b75fa62b0f 2024-11-27T21:34:44,984 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/89d3d6af96b2444bbac6d6b75fa62b0f, entries=150, sequenceid=431, filesize=12.0 K 2024-11-27T21:34:44,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/60527239776b43c8894658bb85bc2a4a as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/60527239776b43c8894658bb85bc2a4a 2024-11-27T21:34:44,996 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/60527239776b43c8894658bb85bc2a4a, entries=150, sequenceid=431, filesize=12.0 K 2024-11-27T21:34:44,997 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for ebbbf218e8d25371b122092896fed576 in 523ms, sequenceid=431, compaction requested=true 2024-11-27T21:34:44,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:44,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:44,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-27T21:34:44,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-27T21:34:45,003 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-27T21:34:45,003 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 991 msec 2024-11-27T21:34:45,005 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 998 msec 2024-11-27T21:34:45,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-27T21:34:45,144 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:34:45,144 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:34:45,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-27T21:34:45,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-27T21:34:45,149 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:34:45,150 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:34:45,150 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:34:45,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on ebbbf218e8d25371b122092896fed576 2024-11-27T21:34:45,189 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:34:45,189 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ebbbf218e8d25371b122092896fed576 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-27T21:34:45,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/6aae2eb4a00b458cb4b020757fb45563 is 50, key is test_row_0/A:col10/1732743285189/Put/seqid=0 2024-11-27T21:34:45,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:45,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743345194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:45,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:45,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:45,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:45,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:45,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743345199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:45,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:45,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:45,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:45,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:45,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743345199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:45,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:45,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:45,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:45,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:45,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743345201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:45,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:45,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:45,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:45,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:45,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743345201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:45,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:45,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:45,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:45,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741929_1105 (size=17181) 2024-11-27T21:34:45,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/6aae2eb4a00b458cb4b020757fb45563 2024-11-27T21:34:45,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/fb8d375f43c844da96580fb08cddb604 is 50, key is test_row_0/B:col10/1732743285189/Put/seqid=0 2024-11-27T21:34:45,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741930_1106 (size=12301) 2024-11-27T21:34:45,242 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/fb8d375f43c844da96580fb08cddb604 2024-11-27T21:34:45,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-27T21:34:45,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/48dced978ad14ed5bc0340ad6d50055d is 50, key is test_row_0/C:col10/1732743285189/Put/seqid=0 2024-11-27T21:34:45,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741931_1107 (size=12301) 2024-11-27T21:34:45,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-27T21:34:45,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:45,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:45,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:45,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:45,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:45,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:45,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:45,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743345313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:45,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:45,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:45,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:45,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743345313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:45,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:45,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:45,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:45,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:45,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743345314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:45,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:45,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:45,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:45,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:45,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743345314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:45,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:45,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:45,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:45,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:45,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-27T21:34:45,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:45,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:45,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:45,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:45,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:45,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:45,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-27T21:34:45,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:45,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743345524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:45,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:45,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:45,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:45,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743345524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:45,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:45,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:45,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:45,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:45,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:45,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743345524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:45,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:45,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:45,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:45,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:45,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743345524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:45,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:45,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:45,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:45,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-27T21:34:45,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:45,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:45,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:45,614 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:45,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:45,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:45,680 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/48dced978ad14ed5bc0340ad6d50055d 2024-11-27T21:34:45,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/6aae2eb4a00b458cb4b020757fb45563 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/6aae2eb4a00b458cb4b020757fb45563 2024-11-27T21:34:45,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/6aae2eb4a00b458cb4b020757fb45563, entries=250, sequenceid=461, filesize=16.8 K 2024-11-27T21:34:45,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/fb8d375f43c844da96580fb08cddb604 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/fb8d375f43c844da96580fb08cddb604 2024-11-27T21:34:45,704 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/fb8d375f43c844da96580fb08cddb604, entries=150, sequenceid=461, filesize=12.0 K 2024-11-27T21:34:45,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/48dced978ad14ed5bc0340ad6d50055d as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/48dced978ad14ed5bc0340ad6d50055d 2024-11-27T21:34:45,713 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/48dced978ad14ed5bc0340ad6d50055d, entries=150, sequenceid=461, filesize=12.0 K 2024-11-27T21:34:45,715 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for ebbbf218e8d25371b122092896fed576 in 526ms, sequenceid=461, compaction requested=true 2024-11-27T21:34:45,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:45,715 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:34:45,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebbbf218e8d25371b122092896fed576:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:34:45,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:34:45,716 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:34:45,718 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55038 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:34:45,718 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): ebbbf218e8d25371b122092896fed576/A is initiating minor compaction (all files) 2024-11-27T21:34:45,718 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebbbf218e8d25371b122092896fed576/A in TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:45,718 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/b645eec5b9fb46ddbe7f396a7b3ad4ec, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/bade09d1cec64c30ab8ed239c09661ed, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/0d49f2abcf6d4baab5543e79008a6265, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/6aae2eb4a00b458cb4b020757fb45563] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp, totalSize=53.7 K 2024-11-27T21:34:45,718 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:34:45,718 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): ebbbf218e8d25371b122092896fed576/B is initiating minor compaction (all files) 2024-11-27T21:34:45,719 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebbbf218e8d25371b122092896fed576/B in TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:45,719 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/21badf024c25408cb484ee62eaa61569, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/d667b1fbf6b34c92a61749030ec94121, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/89d3d6af96b2444bbac6d6b75fa62b0f, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/fb8d375f43c844da96580fb08cddb604] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp, totalSize=49.0 K 2024-11-27T21:34:45,719 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting b645eec5b9fb46ddbe7f396a7b3ad4ec, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1732743283557 2024-11-27T21:34:45,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebbbf218e8d25371b122092896fed576:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:34:45,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:34:45,719 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 21badf024c25408cb484ee62eaa61569, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1732743283557 2024-11-27T21:34:45,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebbbf218e8d25371b122092896fed576:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:34:45,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:34:45,720 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting d667b1fbf6b34c92a61749030ec94121, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1732743283740 2024-11-27T21:34:45,720 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting bade09d1cec64c30ab8ed239c09661ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1732743283740 2024-11-27T21:34:45,721 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 89d3d6af96b2444bbac6d6b75fa62b0f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1732743283878 2024-11-27T21:34:45,721 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d49f2abcf6d4baab5543e79008a6265, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1732743283878 2024-11-27T21:34:45,721 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting fb8d375f43c844da96580fb08cddb604, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1732743284555 2024-11-27T21:34:45,721 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6aae2eb4a00b458cb4b020757fb45563, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1732743284548 2024-11-27T21:34:45,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on ebbbf218e8d25371b122092896fed576 2024-11-27T21:34:45,731 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:34:45,731 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ebbbf218e8d25371b122092896fed576 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T21:34:45,738 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebbbf218e8d25371b122092896fed576#A#compaction#95 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:34:45,739 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebbbf218e8d25371b122092896fed576#B#compaction#96 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:34:45,739 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/c9c786d2080d44e998e75dc73d451ba3 is 50, key is test_row_0/A:col10/1732743285189/Put/seqid=0 2024-11-27T21:34:45,740 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/864e0b875f7d4726aaabf76276024741 is 50, key is test_row_0/B:col10/1732743285189/Put/seqid=0 2024-11-27T21:34:45,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/322ffc92df9a4c4ab6d6b361c469afb5 is 50, key is test_row_0/A:col10/1732743285729/Put/seqid=0 2024-11-27T21:34:45,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-27T21:34:45,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:45,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:45,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:45,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:45,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:45,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:45,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741932_1108 (size=13391) 2024-11-27T21:34:45,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741933_1109 (size=13391) 2024-11-27T21:34:45,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-27T21:34:45,780 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/c9c786d2080d44e998e75dc73d451ba3 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/c9c786d2080d44e998e75dc73d451ba3 2024-11-27T21:34:45,788 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ebbbf218e8d25371b122092896fed576/A of ebbbf218e8d25371b122092896fed576 into c9c786d2080d44e998e75dc73d451ba3(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:34:45,788 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:45,788 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., storeName=ebbbf218e8d25371b122092896fed576/A, priority=12, startTime=1732743285715; duration=0sec 2024-11-27T21:34:45,788 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:34:45,788 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebbbf218e8d25371b122092896fed576:A 2024-11-27T21:34:45,789 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:34:45,791 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:34:45,791 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): ebbbf218e8d25371b122092896fed576/C is initiating minor compaction (all files) 2024-11-27T21:34:45,791 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebbbf218e8d25371b122092896fed576/C in TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:45,791 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/2b38bcf7829f4bd6a1dbee4b1408135a, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/974899bfd3294a46ace9eb40c7c58472, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/60527239776b43c8894658bb85bc2a4a, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/48dced978ad14ed5bc0340ad6d50055d] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp, totalSize=48.9 K 2024-11-27T21:34:45,792 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b38bcf7829f4bd6a1dbee4b1408135a, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1732743283557 2024-11-27T21:34:45,792 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 974899bfd3294a46ace9eb40c7c58472, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1732743283740 2024-11-27T21:34:45,793 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60527239776b43c8894658bb85bc2a4a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1732743283878 2024-11-27T21:34:45,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741934_1110 (size=14741) 2024-11-27T21:34:45,794 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48dced978ad14ed5bc0340ad6d50055d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1732743284555 2024-11-27T21:34:45,814 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebbbf218e8d25371b122092896fed576#C#compaction#98 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:34:45,815 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/7dea71ca2b2b4c40a417055baadb1bb6 is 50, key is test_row_0/C:col10/1732743285189/Put/seqid=0 2024-11-27T21:34:45,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741935_1111 (size=13357) 2024-11-27T21:34:45,839 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/7dea71ca2b2b4c40a417055baadb1bb6 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/7dea71ca2b2b4c40a417055baadb1bb6 2024-11-27T21:34:45,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:45,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743345840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:45,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:45,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:45,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:45,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:45,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743345840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:45,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:45,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743345842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:45,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:45,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:45,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:45,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:45,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:45,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:45,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:45,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743345844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:45,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:45,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:45,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:45,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:45,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743345846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:45,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:45,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:45,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:45,851 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ebbbf218e8d25371b122092896fed576/C of ebbbf218e8d25371b122092896fed576 into 7dea71ca2b2b4c40a417055baadb1bb6(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:34:45,851 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:45,852 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., storeName=ebbbf218e8d25371b122092896fed576/C, priority=12, startTime=1732743285719; duration=0sec 2024-11-27T21:34:45,852 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:34:45,852 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebbbf218e8d25371b122092896fed576:C 2024-11-27T21:34:45,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-27T21:34:45,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:45,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:45,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:45,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:45,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:45,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:45,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:45,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743345954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:45,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:45,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:45,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:45,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743345954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:45,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:45,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:45,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743345954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:45,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:45,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:45,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:45,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:45,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:45,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743345954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:45,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:45,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:45,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:45,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:45,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:45,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743345954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:45,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:45,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:45,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:45,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:46,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-27T21:34:46,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:46,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:46,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:46,076 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:46,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:46,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:46,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:46,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743346164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:46,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:46,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:46,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:46,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:46,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743346164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:46,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:46,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:46,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:46,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:46,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743346164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:46,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:46,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743346164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:46,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:46,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:46,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:46,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:46,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:46,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:46,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:46,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743346174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:46,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:46,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:46,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:46,180 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/864e0b875f7d4726aaabf76276024741 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/864e0b875f7d4726aaabf76276024741 2024-11-27T21:34:46,187 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ebbbf218e8d25371b122092896fed576/B of ebbbf218e8d25371b122092896fed576 into 864e0b875f7d4726aaabf76276024741(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:34:46,187 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:46,187 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., storeName=ebbbf218e8d25371b122092896fed576/B, priority=12, startTime=1732743285716; duration=0sec 2024-11-27T21:34:46,187 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:34:46,187 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebbbf218e8d25371b122092896fed576:B 2024-11-27T21:34:46,195 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/322ffc92df9a4c4ab6d6b361c469afb5 2024-11-27T21:34:46,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/886e88fdbdb84cd5b1ad55975664aeac is 50, key is test_row_0/B:col10/1732743285729/Put/seqid=0 2024-11-27T21:34:46,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741936_1112 (size=12301) 2024-11-27T21:34:46,211 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/886e88fdbdb84cd5b1ad55975664aeac 2024-11-27T21:34:46,219 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/316625f62c7141e3a71292232b41f72e is 50, key is test_row_0/C:col10/1732743285729/Put/seqid=0 2024-11-27T21:34:46,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741937_1113 (size=12301) 2024-11-27T21:34:46,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-27T21:34:46,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:46,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:46,229 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/316625f62c7141e3a71292232b41f72e 2024-11-27T21:34:46,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:46,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:46,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:46,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:46,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/322ffc92df9a4c4ab6d6b361c469afb5 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/322ffc92df9a4c4ab6d6b361c469afb5 2024-11-27T21:34:46,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/322ffc92df9a4c4ab6d6b361c469afb5, entries=200, sequenceid=472, filesize=14.4 K 2024-11-27T21:34:46,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/886e88fdbdb84cd5b1ad55975664aeac as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/886e88fdbdb84cd5b1ad55975664aeac 2024-11-27T21:34:46,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/886e88fdbdb84cd5b1ad55975664aeac, entries=150, sequenceid=472, filesize=12.0 K 2024-11-27T21:34:46,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/316625f62c7141e3a71292232b41f72e as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/316625f62c7141e3a71292232b41f72e 2024-11-27T21:34:46,261 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/316625f62c7141e3a71292232b41f72e, entries=150, sequenceid=472, filesize=12.0 K 2024-11-27T21:34:46,262 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for ebbbf218e8d25371b122092896fed576 in 531ms, sequenceid=472, compaction requested=false 2024-11-27T21:34:46,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:46,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-27T21:34:46,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-27T21:34:46,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:46,383 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing ebbbf218e8d25371b122092896fed576 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T21:34:46,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/6e1a1c5705be458cb0ddf6d17c064ee6 is 50, key is test_row_0/A:col10/1732743285838/Put/seqid=0 2024-11-27T21:34:46,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741938_1114 (size=12301) 2024-11-27T21:34:46,406 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/6e1a1c5705be458cb0ddf6d17c064ee6 2024-11-27T21:34:46,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/b1a961c1743b45948578b98774ff1900 is 50, key is test_row_0/B:col10/1732743285838/Put/seqid=0 2024-11-27T21:34:46,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741939_1115 (size=12301) 2024-11-27T21:34:46,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:46,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on ebbbf218e8d25371b122092896fed576 2024-11-27T21:34:46,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:46,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:46,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743346481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:46,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743346480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:46,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:46,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:46,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:46,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:46,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:46,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:46,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:46,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743346484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:46,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:46,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:46,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:46,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:46,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743346487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:46,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:46,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743346487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:46,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:46,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:46,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:46,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:46,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:46,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:46,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:46,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743346594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:46,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:46,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:46,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:46,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743346594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:46,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:46,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:46,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743346594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:46,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:46,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:46,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:46,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:46,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:46,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:46,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:46,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743346603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:46,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:46,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:46,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:46,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:46,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743346803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:46,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:46,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:46,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743346803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:46,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:46,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:46,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:46,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743346803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:46,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:46,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:46,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:46,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:46,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:46,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:46,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:46,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743346813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:46,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:46,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:46,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:46,863 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/b1a961c1743b45948578b98774ff1900 2024-11-27T21:34:46,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/7e7f01f7a0e0427bb076455cb5b27139 is 50, key is test_row_0/C:col10/1732743285838/Put/seqid=0 2024-11-27T21:34:46,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741940_1116 (size=12301) 2024-11-27T21:34:47,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:47,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743347003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:47,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:47,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:47,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:47,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:47,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743347114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:47,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:47,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743347114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:47,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:47,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:47,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:47,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:47,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743347114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:47,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:47,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:47,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:47,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:47,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:47,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:47,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:47,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743347123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:47,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:47,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:47,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:47,287 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/7e7f01f7a0e0427bb076455cb5b27139 2024-11-27T21:34:47,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/6e1a1c5705be458cb0ddf6d17c064ee6 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/6e1a1c5705be458cb0ddf6d17c064ee6 2024-11-27T21:34:47,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-27T21:34:47,298 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/6e1a1c5705be458cb0ddf6d17c064ee6, entries=150, sequenceid=500, filesize=12.0 K 2024-11-27T21:34:47,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/b1a961c1743b45948578b98774ff1900 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/b1a961c1743b45948578b98774ff1900 2024-11-27T21:34:47,304 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/b1a961c1743b45948578b98774ff1900, entries=150, sequenceid=500, filesize=12.0 K 2024-11-27T21:34:47,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/7e7f01f7a0e0427bb076455cb5b27139 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/7e7f01f7a0e0427bb076455cb5b27139 2024-11-27T21:34:47,309 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/7e7f01f7a0e0427bb076455cb5b27139, entries=150, sequenceid=500, filesize=12.0 K 2024-11-27T21:34:47,310 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for ebbbf218e8d25371b122092896fed576 in 927ms, sequenceid=500, compaction requested=true 2024-11-27T21:34:47,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:47,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:47,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-27T21:34:47,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-27T21:34:47,314 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-27T21:34:47,314 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1620 sec 2024-11-27T21:34:47,316 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 2.1700 sec 2024-11-27T21:34:47,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on ebbbf218e8d25371b122092896fed576 2024-11-27T21:34:47,625 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:34:47,625 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ebbbf218e8d25371b122092896fed576 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-27T21:34:47,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/fc8af206cac24a6fb62a569308746f81 is 50, key is test_row_0/A:col10/1732743287625/Put/seqid=0 2024-11-27T21:34:47,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741941_1117 (size=12301) 2024-11-27T21:34:47,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:47,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743347657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:47,664 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:47,664 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:47,664 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:47,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:47,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743347661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:47,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:47,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743347662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:47,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:47,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743347664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:47,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:47,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:47,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:47,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:47,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:47,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:47,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:47,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:47,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:47,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:47,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743347773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:47,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:47,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:47,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:47,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:47,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743347774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:47,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:47,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:47,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:47,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:47,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743347774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:47,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:47,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:47,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:47,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:47,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743347774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:47,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:47,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:47,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:47,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:47,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743347984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:47,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:47,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:47,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743347984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:47,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:47,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:47,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:47,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:47,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:47,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:47,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743347984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:47,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:47,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:47,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:47,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:47,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743347984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:47,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:47,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:47,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:48,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:48,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743348023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:48,031 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,39097,1732743267048 for 'test_row_0' in ebbbf218e8d25371b122092896fed576 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:34:48,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:48,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:48,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:48,058 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=514 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/fc8af206cac24a6fb62a569308746f81 2024-11-27T21:34:48,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/165b8c4eb3b7414eae6551cd99a87b48 is 50, key is test_row_0/B:col10/1732743287625/Put/seqid=0 2024-11-27T21:34:48,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741942_1118 (size=12301) 2024-11-27T21:34:48,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:48,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743348293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:48,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:48,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:48,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:48,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:48,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743348294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:48,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:48,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:48,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743348294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:48,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:48,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:48,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:48,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743348294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:48,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:48,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:48,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:48,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:48,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:48,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:48,470 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=514 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/165b8c4eb3b7414eae6551cd99a87b48 2024-11-27T21:34:48,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/54e059546a0e4e329c821e0efd59ff29 is 50, key is test_row_0/C:col10/1732743287625/Put/seqid=0 2024-11-27T21:34:48,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741943_1119 (size=12301) 2024-11-27T21:34:48,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:48,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743348803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:48,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:48,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:48,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743348803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:48,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:48,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:48,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:48,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:48,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:48,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743348804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:48,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:48,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:48,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743348804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:48,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:48,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:48,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:48,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:48,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:48,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:48,886 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=514 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/54e059546a0e4e329c821e0efd59ff29 2024-11-27T21:34:48,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/fc8af206cac24a6fb62a569308746f81 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/fc8af206cac24a6fb62a569308746f81 2024-11-27T21:34:48,897 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/fc8af206cac24a6fb62a569308746f81, entries=150, sequenceid=514, filesize=12.0 K 2024-11-27T21:34:48,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/165b8c4eb3b7414eae6551cd99a87b48 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/165b8c4eb3b7414eae6551cd99a87b48 2024-11-27T21:34:48,902 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/165b8c4eb3b7414eae6551cd99a87b48, entries=150, sequenceid=514, filesize=12.0 K 2024-11-27T21:34:48,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/54e059546a0e4e329c821e0efd59ff29 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/54e059546a0e4e329c821e0efd59ff29 2024-11-27T21:34:48,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/54e059546a0e4e329c821e0efd59ff29, entries=150, sequenceid=514, filesize=12.0 K 2024-11-27T21:34:48,911 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for ebbbf218e8d25371b122092896fed576 in 1286ms, sequenceid=514, compaction requested=true 2024-11-27T21:34:48,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:48,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebbbf218e8d25371b122092896fed576:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:34:48,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:34:48,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebbbf218e8d25371b122092896fed576:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:34:48,911 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:34:48,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:34:48,911 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:34:48,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebbbf218e8d25371b122092896fed576:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:34:48,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:34:48,913 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50294 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:34:48,913 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52734 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:34:48,913 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): ebbbf218e8d25371b122092896fed576/B is initiating minor compaction (all files) 2024-11-27T21:34:48,913 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): ebbbf218e8d25371b122092896fed576/A is initiating minor compaction (all files) 2024-11-27T21:34:48,913 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebbbf218e8d25371b122092896fed576/B in TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:48,914 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebbbf218e8d25371b122092896fed576/A in TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:48,914 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/c9c786d2080d44e998e75dc73d451ba3, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/322ffc92df9a4c4ab6d6b361c469afb5, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/6e1a1c5705be458cb0ddf6d17c064ee6, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/fc8af206cac24a6fb62a569308746f81] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp, totalSize=51.5 K 2024-11-27T21:34:48,914 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/864e0b875f7d4726aaabf76276024741, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/886e88fdbdb84cd5b1ad55975664aeac, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/b1a961c1743b45948578b98774ff1900, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/165b8c4eb3b7414eae6551cd99a87b48] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp, totalSize=49.1 K 2024-11-27T21:34:48,914 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 864e0b875f7d4726aaabf76276024741, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1732743284555 2024-11-27T21:34:48,914 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9c786d2080d44e998e75dc73d451ba3, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1732743284555 2024-11-27T21:34:48,915 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 886e88fdbdb84cd5b1ad55975664aeac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1732743285198 2024-11-27T21:34:48,915 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 322ffc92df9a4c4ab6d6b361c469afb5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1732743285198 2024-11-27T21:34:48,915 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting b1a961c1743b45948578b98774ff1900, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=500, earliestPutTs=1732743285838 2024-11-27T21:34:48,915 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e1a1c5705be458cb0ddf6d17c064ee6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=500, earliestPutTs=1732743285838 2024-11-27T21:34:48,916 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 165b8c4eb3b7414eae6551cd99a87b48, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=514, earliestPutTs=1732743287623 2024-11-27T21:34:48,916 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc8af206cac24a6fb62a569308746f81, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=514, earliestPutTs=1732743287623 2024-11-27T21:34:48,929 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebbbf218e8d25371b122092896fed576#B#compaction#107 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:34:48,929 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebbbf218e8d25371b122092896fed576#A#compaction#108 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:34:48,930 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/f898192e973a4318b1556db945e032eb is 50, key is test_row_0/B:col10/1732743287625/Put/seqid=0 2024-11-27T21:34:48,930 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/15c3cdb202c74d968d35b5947e91d6d9 is 50, key is test_row_0/A:col10/1732743287625/Put/seqid=0 2024-11-27T21:34:48,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741944_1120 (size=13527) 2024-11-27T21:34:48,943 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/f898192e973a4318b1556db945e032eb as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/f898192e973a4318b1556db945e032eb 2024-11-27T21:34:48,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741945_1121 (size=13527) 2024-11-27T21:34:48,953 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ebbbf218e8d25371b122092896fed576/B of ebbbf218e8d25371b122092896fed576 into f898192e973a4318b1556db945e032eb(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:34:48,953 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:48,953 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., storeName=ebbbf218e8d25371b122092896fed576/B, priority=12, startTime=1732743288911; duration=0sec 2024-11-27T21:34:48,953 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:34:48,953 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebbbf218e8d25371b122092896fed576:B 2024-11-27T21:34:48,954 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:34:48,955 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/15c3cdb202c74d968d35b5947e91d6d9 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/15c3cdb202c74d968d35b5947e91d6d9 2024-11-27T21:34:48,956 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50260 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:34:48,956 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): ebbbf218e8d25371b122092896fed576/C is initiating minor compaction (all files) 2024-11-27T21:34:48,956 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebbbf218e8d25371b122092896fed576/C in TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:48,956 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/7dea71ca2b2b4c40a417055baadb1bb6, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/316625f62c7141e3a71292232b41f72e, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/7e7f01f7a0e0427bb076455cb5b27139, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/54e059546a0e4e329c821e0efd59ff29] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp, totalSize=49.1 K 2024-11-27T21:34:48,957 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 7dea71ca2b2b4c40a417055baadb1bb6, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1732743284555 2024-11-27T21:34:48,957 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 316625f62c7141e3a71292232b41f72e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1732743285198 2024-11-27T21:34:48,958 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e7f01f7a0e0427bb076455cb5b27139, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=500, earliestPutTs=1732743285838 2024-11-27T21:34:48,959 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 54e059546a0e4e329c821e0efd59ff29, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=514, earliestPutTs=1732743287623 2024-11-27T21:34:48,963 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ebbbf218e8d25371b122092896fed576/A of ebbbf218e8d25371b122092896fed576 into 15c3cdb202c74d968d35b5947e91d6d9(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:34:48,963 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:48,964 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., storeName=ebbbf218e8d25371b122092896fed576/A, priority=12, startTime=1732743288911; duration=0sec 2024-11-27T21:34:48,964 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:34:48,964 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebbbf218e8d25371b122092896fed576:A 2024-11-27T21:34:48,972 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebbbf218e8d25371b122092896fed576#C#compaction#109 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:34:48,972 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/0bf415e6893a44fdba6054db257e6b28 is 50, key is test_row_0/C:col10/1732743287625/Put/seqid=0 2024-11-27T21:34:48,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741946_1122 (size=13493) 2024-11-27T21:34:48,987 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/0bf415e6893a44fdba6054db257e6b28 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/0bf415e6893a44fdba6054db257e6b28 2024-11-27T21:34:48,993 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ebbbf218e8d25371b122092896fed576/C of ebbbf218e8d25371b122092896fed576 into 0bf415e6893a44fdba6054db257e6b28(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:34:48,993 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:48,993 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., storeName=ebbbf218e8d25371b122092896fed576/C, priority=12, startTime=1732743288911; duration=0sec 2024-11-27T21:34:48,994 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:34:48,994 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebbbf218e8d25371b122092896fed576:C 2024-11-27T21:34:49,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-27T21:34:49,314 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:34:49,314 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:34:49,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-27T21:34:49,317 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:34:49,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-27T21:34:49,318 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:34:49,318 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:34:49,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-27T21:34:49,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-27T21:34:49,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:49,472 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing ebbbf218e8d25371b122092896fed576 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-27T21:34:49,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/b4dd9786520a4fb89d17b66f6bab8067 is 50, key is test_row_0/A:col10/1732743287653/Put/seqid=0 2024-11-27T21:34:49,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741947_1123 (size=12301) 2024-11-27T21:34:49,489 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=539 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/b4dd9786520a4fb89d17b66f6bab8067 2024-11-27T21:34:49,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/11647a6c65f54a1d909157fc08977f61 is 50, key is test_row_0/B:col10/1732743287653/Put/seqid=0 2024-11-27T21:34:49,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741948_1124 (size=12301) 2024-11-27T21:34:49,509 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=539 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/11647a6c65f54a1d909157fc08977f61 2024-11-27T21:34:49,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/329dfe9276d24538bc65472253c7e400 is 50, key is test_row_0/C:col10/1732743287653/Put/seqid=0 2024-11-27T21:34:49,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741949_1125 (size=12301) 2024-11-27T21:34:49,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-27T21:34:49,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on ebbbf218e8d25371b122092896fed576 2024-11-27T21:34:49,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:49,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:49,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743349836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:49,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:49,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:49,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:49,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:49,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743349839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:49,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:49,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:49,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:49,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:49,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743349840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:49,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:49,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:49,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743349841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:49,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:49,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:49,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:49,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:49,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:49,931 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=539 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/329dfe9276d24538bc65472253c7e400 2024-11-27T21:34:49,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/b4dd9786520a4fb89d17b66f6bab8067 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/b4dd9786520a4fb89d17b66f6bab8067 2024-11-27T21:34:49,942 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/b4dd9786520a4fb89d17b66f6bab8067, entries=150, sequenceid=539, filesize=12.0 K 2024-11-27T21:34:49,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-27T21:34:49,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:49,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743349943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:49,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:49,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:49,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:49,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:49,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743349947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:49,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:49,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:49,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:49,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/11647a6c65f54a1d909157fc08977f61 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/11647a6c65f54a1d909157fc08977f61 2024-11-27T21:34:49,954 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/11647a6c65f54a1d909157fc08977f61, entries=150, sequenceid=539, filesize=12.0 K 2024-11-27T21:34:49,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/329dfe9276d24538bc65472253c7e400 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/329dfe9276d24538bc65472253c7e400 2024-11-27T21:34:49,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:49,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:49,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743349954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:49,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743349954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:49,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:49,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:49,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:49,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:49,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:49,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:49,962 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/329dfe9276d24538bc65472253c7e400, entries=150, sequenceid=539, filesize=12.0 K 2024-11-27T21:34:49,963 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for ebbbf218e8d25371b122092896fed576 in 492ms, sequenceid=539, compaction requested=false 2024-11-27T21:34:49,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:49,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:49,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-27T21:34:49,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-27T21:34:49,967 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-27T21:34:49,967 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 646 msec 2024-11-27T21:34:49,969 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 653 msec 2024-11-27T21:34:50,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on ebbbf218e8d25371b122092896fed576 2024-11-27T21:34:50,056 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:34:50,056 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ebbbf218e8d25371b122092896fed576 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-27T21:34:50,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/b3d1a9e9ab1541beba2e2a1a175624da is 50, key is test_row_0/A:col10/1732743290054/Put/seqid=0 2024-11-27T21:34:50,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741950_1126 (size=14741) 2024-11-27T21:34:50,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:50,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743350116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:50,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:50,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:50,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:50,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:50,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743350153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:50,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:50,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743350153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:50,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:50,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:50,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:50,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:50,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743350164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:50,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:50,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:50,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:50,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743350164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:50,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:50,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:50,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:50,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:50,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:50,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743350223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:50,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:50,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:50,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:50,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:50,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743350433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:50,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:50,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:50,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:50,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-27T21:34:50,453 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:34:50,454 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:34:50,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-27T21:34:50,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-27T21:34:50,456 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:34:50,456 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:34:50,457 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:34:50,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:50,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743350463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:50,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:50,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:50,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:50,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743350464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:50,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:50,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:50,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:50,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:50,470 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=555 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/b3d1a9e9ab1541beba2e2a1a175624da 2024-11-27T21:34:50,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:50,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743350473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:50,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:50,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:50,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:50,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:50,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743350473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:50,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:50,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:50,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:50,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/ef68b765210d450fbeef23e1a2c5e15a is 50, key is test_row_0/B:col10/1732743290054/Put/seqid=0 2024-11-27T21:34:50,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741951_1127 (size=12301) 2024-11-27T21:34:50,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-27T21:34:50,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-27T21:34:50,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:50,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:50,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:50,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:50,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:50,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:50,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:50,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743350743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:50,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:50,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:50,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:50,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-27T21:34:50,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:50,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:50,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:50,763 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:50,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:50,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:50,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-27T21:34:50,884 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=555 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/ef68b765210d450fbeef23e1a2c5e15a 2024-11-27T21:34:50,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/9b625497959b4b3b8ed7fd101ad3dd1f is 50, key is test_row_0/C:col10/1732743290054/Put/seqid=0 2024-11-27T21:34:50,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741952_1128 (size=12301) 2024-11-27T21:34:50,899 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=555 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/9b625497959b4b3b8ed7fd101ad3dd1f 2024-11-27T21:34:50,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/b3d1a9e9ab1541beba2e2a1a175624da as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/b3d1a9e9ab1541beba2e2a1a175624da 2024-11-27T21:34:50,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/b3d1a9e9ab1541beba2e2a1a175624da, entries=200, sequenceid=555, filesize=14.4 K 2024-11-27T21:34:50,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/ef68b765210d450fbeef23e1a2c5e15a as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/ef68b765210d450fbeef23e1a2c5e15a 2024-11-27T21:34:50,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-27T21:34:50,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:50,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:50,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:50,916 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:50,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:50,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/ef68b765210d450fbeef23e1a2c5e15a, entries=150, sequenceid=555, filesize=12.0 K 2024-11-27T21:34:50,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:50,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/9b625497959b4b3b8ed7fd101ad3dd1f as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/9b625497959b4b3b8ed7fd101ad3dd1f 2024-11-27T21:34:50,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/9b625497959b4b3b8ed7fd101ad3dd1f, entries=150, sequenceid=555, filesize=12.0 K 2024-11-27T21:34:50,926 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for ebbbf218e8d25371b122092896fed576 in 870ms, sequenceid=555, compaction requested=true 2024-11-27T21:34:50,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:50,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebbbf218e8d25371b122092896fed576:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:34:50,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:34:50,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebbbf218e8d25371b122092896fed576:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:34:50,926 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:34:50,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:34:50,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebbbf218e8d25371b122092896fed576:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:34:50,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-27T21:34:50,926 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:34:50,927 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40569 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:34:50,927 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): ebbbf218e8d25371b122092896fed576/A is initiating minor compaction (all files) 2024-11-27T21:34:50,928 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebbbf218e8d25371b122092896fed576/A in TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:50,928 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/15c3cdb202c74d968d35b5947e91d6d9, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/b4dd9786520a4fb89d17b66f6bab8067, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/b3d1a9e9ab1541beba2e2a1a175624da] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp, totalSize=39.6 K 2024-11-27T21:34:50,928 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38129 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:34:50,928 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): ebbbf218e8d25371b122092896fed576/B is initiating minor compaction (all files) 2024-11-27T21:34:50,928 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebbbf218e8d25371b122092896fed576/B in TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:50,928 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/f898192e973a4318b1556db945e032eb, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/11647a6c65f54a1d909157fc08977f61, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/ef68b765210d450fbeef23e1a2c5e15a] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp, totalSize=37.2 K 2024-11-27T21:34:50,929 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15c3cdb202c74d968d35b5947e91d6d9, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=514, earliestPutTs=1732743287623 2024-11-27T21:34:50,929 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting f898192e973a4318b1556db945e032eb, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=514, earliestPutTs=1732743287623 2024-11-27T21:34:50,929 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4dd9786520a4fb89d17b66f6bab8067, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=539, earliestPutTs=1732743287653 2024-11-27T21:34:50,930 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 11647a6c65f54a1d909157fc08977f61, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=539, earliestPutTs=1732743287653 2024-11-27T21:34:50,930 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3d1a9e9ab1541beba2e2a1a175624da, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=555, earliestPutTs=1732743289834 2024-11-27T21:34:50,930 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting ef68b765210d450fbeef23e1a2c5e15a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=555, earliestPutTs=1732743289838 2024-11-27T21:34:50,940 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebbbf218e8d25371b122092896fed576#B#compaction#116 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:34:50,941 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/dfc82f120cf04a82864dc1d42d00d6f5 is 50, key is test_row_0/B:col10/1732743290054/Put/seqid=0 2024-11-27T21:34:50,943 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebbbf218e8d25371b122092896fed576#A#compaction#117 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:34:50,944 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/b93b06a5362545ffa3041ab67cded9f9 is 50, key is test_row_0/A:col10/1732743290054/Put/seqid=0 2024-11-27T21:34:50,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741954_1130 (size=13629) 2024-11-27T21:34:50,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741953_1129 (size=13629) 2024-11-27T21:34:50,959 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/b93b06a5362545ffa3041ab67cded9f9 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/b93b06a5362545ffa3041ab67cded9f9 2024-11-27T21:34:50,962 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/dfc82f120cf04a82864dc1d42d00d6f5 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/dfc82f120cf04a82864dc1d42d00d6f5 2024-11-27T21:34:50,966 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebbbf218e8d25371b122092896fed576/A of ebbbf218e8d25371b122092896fed576 into b93b06a5362545ffa3041ab67cded9f9(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:34:50,966 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:50,966 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., storeName=ebbbf218e8d25371b122092896fed576/A, priority=13, startTime=1732743290926; duration=0sec 2024-11-27T21:34:50,967 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:34:50,967 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebbbf218e8d25371b122092896fed576:A 2024-11-27T21:34:50,967 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:34:50,968 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:34:50,968 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): ebbbf218e8d25371b122092896fed576/C is initiating minor compaction (all files) 2024-11-27T21:34:50,968 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebbbf218e8d25371b122092896fed576/C in TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:50,968 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/0bf415e6893a44fdba6054db257e6b28, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/329dfe9276d24538bc65472253c7e400, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/9b625497959b4b3b8ed7fd101ad3dd1f] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp, totalSize=37.2 K 2024-11-27T21:34:50,968 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebbbf218e8d25371b122092896fed576/B of ebbbf218e8d25371b122092896fed576 into dfc82f120cf04a82864dc1d42d00d6f5(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:34:50,968 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:50,968 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., storeName=ebbbf218e8d25371b122092896fed576/B, priority=13, startTime=1732743290926; duration=0sec 2024-11-27T21:34:50,969 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bf415e6893a44fdba6054db257e6b28, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=514, earliestPutTs=1732743287623 2024-11-27T21:34:50,969 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:34:50,969 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebbbf218e8d25371b122092896fed576:B 2024-11-27T21:34:50,969 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 329dfe9276d24538bc65472253c7e400, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=539, earliestPutTs=1732743287653 2024-11-27T21:34:50,970 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b625497959b4b3b8ed7fd101ad3dd1f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=555, earliestPutTs=1732743289838 2024-11-27T21:34:50,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on ebbbf218e8d25371b122092896fed576 2024-11-27T21:34:50,977 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:34:50,977 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ebbbf218e8d25371b122092896fed576 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-27T21:34:50,979 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebbbf218e8d25371b122092896fed576#C#compaction#118 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:34:50,979 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/188d731cb47d4910a630993c2fdab611 is 50, key is test_row_0/C:col10/1732743290054/Put/seqid=0 2024-11-27T21:34:50,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/6cd738af8b134d889f4c141e26587a25 is 50, key is test_row_0/A:col10/1732743290114/Put/seqid=0 2024-11-27T21:34:50,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741955_1131 (size=13595) 2024-11-27T21:34:50,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741956_1132 (size=17181) 2024-11-27T21:34:50,991 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=579 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/6cd738af8b134d889f4c141e26587a25 2024-11-27T21:34:51,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/50b7315ba3744a20bafcdbe7b8ac9c59 is 50, key is test_row_0/B:col10/1732743290114/Put/seqid=0 2024-11-27T21:34:51,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:51,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743350995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:51,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:51,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:51,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:51,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:51,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743350995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:51,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:51,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:51,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:51,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:51,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743350997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:51,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:51,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:51,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:51,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:51,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743350999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:51,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:51,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:51,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:51,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741957_1133 (size=12301) 2024-11-27T21:34:51,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-27T21:34:51,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:51,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:51,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:51,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:51,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:51,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:51,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-27T21:34:51,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:51,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743351103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:51,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:51,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:51,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:51,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:51,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743351113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:51,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:51,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:51,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743351113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:51,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:51,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:51,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:51,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:51,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743351113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:51,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:51,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:51,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:51,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:51,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:51,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-27T21:34:51,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:51,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:51,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:51,224 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:51,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:51,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:51,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:51,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36052 deadline: 1732743351254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:51,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:51,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:51,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:51,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:51,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743351313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:51,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:51,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:51,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:51,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:51,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743351324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:51,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:51,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:51,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:51,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:51,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743351324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:51,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:51,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:51,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:51,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743351324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:51,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:51,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:51,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:51,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:51,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-27T21:34:51,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:51,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:51,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:51,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:51,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:51,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:51,395 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/188d731cb47d4910a630993c2fdab611 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/188d731cb47d4910a630993c2fdab611 2024-11-27T21:34:51,401 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebbbf218e8d25371b122092896fed576/C of ebbbf218e8d25371b122092896fed576 into 188d731cb47d4910a630993c2fdab611(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:34:51,401 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:51,401 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., storeName=ebbbf218e8d25371b122092896fed576/C, priority=13, startTime=1732743290926; duration=0sec 2024-11-27T21:34:51,401 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:34:51,401 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebbbf218e8d25371b122092896fed576:C 2024-11-27T21:34:51,406 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=579 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/50b7315ba3744a20bafcdbe7b8ac9c59 2024-11-27T21:34:51,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/aeb21ddae35c432a97215478c9a2f90c is 50, key is test_row_0/C:col10/1732743290114/Put/seqid=0 2024-11-27T21:34:51,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741958_1134 (size=12301) 2024-11-27T21:34:51,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-27T21:34:51,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:51,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:51,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:51,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:51,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:51,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:51,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-27T21:34:51,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1732743351623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:51,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:51,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:51,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:51,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:51,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1732743351633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:51,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:51,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:51,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:51,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36036 deadline: 1732743351633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:51,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:51,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:51,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:34:51,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:51,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:51,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35994 deadline: 1732743351633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 2024-11-27T21:34:51,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:51,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:34:51,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:34:51,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-27T21:34:51,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:51,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:51,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:51,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:51,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:51,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:51,824 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=579 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/aeb21ddae35c432a97215478c9a2f90c 2024-11-27T21:34:51,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/6cd738af8b134d889f4c141e26587a25 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/6cd738af8b134d889f4c141e26587a25 2024-11-27T21:34:51,835 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/6cd738af8b134d889f4c141e26587a25, entries=250, sequenceid=579, filesize=16.8 K 2024-11-27T21:34:51,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-27T21:34:51,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:51,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:51,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:51,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:51,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:51,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/50b7315ba3744a20bafcdbe7b8ac9c59 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/50b7315ba3744a20bafcdbe7b8ac9c59 2024-11-27T21:34:51,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:34:51,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/50b7315ba3744a20bafcdbe7b8ac9c59, entries=150, sequenceid=579, filesize=12.0 K 2024-11-27T21:34:51,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/aeb21ddae35c432a97215478c9a2f90c as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/aeb21ddae35c432a97215478c9a2f90c 2024-11-27T21:34:51,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/aeb21ddae35c432a97215478c9a2f90c, entries=150, sequenceid=579, filesize=12.0 K 2024-11-27T21:34:51,850 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for ebbbf218e8d25371b122092896fed576 in 873ms, sequenceid=579, compaction requested=false 2024-11-27T21:34:51,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:51,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-27T21:34:51,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:51,990 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing ebbbf218e8d25371b122092896fed576 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-27T21:34:51,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/122e4c88acdb45689fd577f427cab249 is 50, key is test_row_0/A:col10/1732743290988/Put/seqid=0 2024-11-27T21:34:52,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741959_1135 (size=12301) 2024-11-27T21:34:52,075 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-27T21:34:52,075 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-27T21:34:52,075 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:34:52,075 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:34:52,076 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-27T21:34:52,076 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:34:52,077 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-27T21:34:52,077 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:34:52,081 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:34:52,081 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:34:52,081 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:34:52,081 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:34:52,081 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:34:52,081 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:34:52,081 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:34:52,081 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:34:52,081 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:34:52,082 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:34:52,082 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:34:52,082 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:34:52,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on ebbbf218e8d25371b122092896fed576 2024-11-27T21:34:52,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. as already flushing 2024-11-27T21:34:52,135 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-27T21:34:52,135 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:34:52,135 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:34:52,135 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:34:52,135 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:34:52,145 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-27T21:34:52,145 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-27T21:34:52,146 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:34:52,146 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:34:52,146 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:34:52,146 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:34:52,146 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:34:52,146 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:34:52,146 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:34:52,146 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:34:52,146 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-27T21:34:52,146 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:34:52,146 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:34:52,146 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:34:52,146 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:34:52,264 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-27T21:34:52,265 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:34:52,265 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:34:52,265 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:34:52,265 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:34:52,401 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=594 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/122e4c88acdb45689fd577f427cab249 2024-11-27T21:34:52,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/2b4c6ecba1ea447eb4d67d351db7e0d0 is 50, key is test_row_0/B:col10/1732743290988/Put/seqid=0 2024-11-27T21:34:52,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741960_1136 (size=12301) 2024-11-27T21:34:52,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-27T21:34:52,813 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=594 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/2b4c6ecba1ea447eb4d67d351db7e0d0 2024-11-27T21:34:52,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/364edfb94d404490bd84db0f5a80f9c9 is 50, key is test_row_0/C:col10/1732743290988/Put/seqid=0 2024-11-27T21:34:52,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741961_1137 (size=12301) 2024-11-27T21:34:53,225 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=594 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/364edfb94d404490bd84db0f5a80f9c9 2024-11-27T21:34:53,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/122e4c88acdb45689fd577f427cab249 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/122e4c88acdb45689fd577f427cab249 2024-11-27T21:34:53,234 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/122e4c88acdb45689fd577f427cab249, entries=150, sequenceid=594, filesize=12.0 K 2024-11-27T21:34:53,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/2b4c6ecba1ea447eb4d67d351db7e0d0 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/2b4c6ecba1ea447eb4d67d351db7e0d0 2024-11-27T21:34:53,239 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/2b4c6ecba1ea447eb4d67d351db7e0d0, entries=150, sequenceid=594, filesize=12.0 K 2024-11-27T21:34:53,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/364edfb94d404490bd84db0f5a80f9c9 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/364edfb94d404490bd84db0f5a80f9c9 2024-11-27T21:34:53,243 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/364edfb94d404490bd84db0f5a80f9c9, entries=150, sequenceid=594, filesize=12.0 K 2024-11-27T21:34:53,244 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=33.54 KB/34350 for ebbbf218e8d25371b122092896fed576 in 1254ms, sequenceid=594, compaction requested=true 2024-11-27T21:34:53,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for ebbbf218e8d25371b122092896fed576: 2024-11-27T21:34:53,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:53,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-27T21:34:53,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-27T21:34:53,247 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-27T21:34:53,247 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7880 sec 2024-11-27T21:34:53,249 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 2.7940 sec 2024-11-27T21:34:54,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-27T21:34:54,613 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:34:54,613 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-27T21:34:54,614 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 101 2024-11-27T21:34:54,614 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 86 2024-11-27T21:34:54,614 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 109 2024-11-27T21:34:54,614 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2024-11-27T21:34:54,614 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 95 2024-11-27T21:34:54,614 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-27T21:34:54,614 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3263 2024-11-27T21:34:54,614 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3232 2024-11-27T21:34:54,614 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-27T21:34:54,614 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1492 2024-11-27T21:34:54,614 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4464 rows 2024-11-27T21:34:54,614 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1488 2024-11-27T21:34:54,614 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4456 rows 2024-11-27T21:34:54,614 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T21:34:54,614 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T21:34:54,614 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:34:54,614 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:34:54,614 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:34:54,618 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-27T21:34:54,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-27T21:34:54,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T21:34:54,625 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732743294625"}]},"ts":"1732743294625"} 2024-11-27T21:34:54,627 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-27T21:34:54,627 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-27T21:34:54,629 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T21:34:54,633 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ebbbf218e8d25371b122092896fed576, UNASSIGN}] 2024-11-27T21:34:54,634 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ebbbf218e8d25371b122092896fed576, UNASSIGN 2024-11-27T21:34:54,635 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=ebbbf218e8d25371b122092896fed576, regionState=CLOSING, regionLocation=4585c39bed03,39097,1732743267048 2024-11-27T21:34:54,637 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ebbbf218e8d25371b122092896fed576, UNASSIGN because future has completed 2024-11-27T21:34:54,638 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T21:34:54,638 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048}] 2024-11-27T21:34:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T21:34:54,794 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close ebbbf218e8d25371b122092896fed576 2024-11-27T21:34:54,795 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T21:34:54,795 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing ebbbf218e8d25371b122092896fed576, disabling compactions & flushes 2024-11-27T21:34:54,795 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:54,795 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:54,795 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. after waiting 0 ms 2024-11-27T21:34:54,795 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:54,795 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2902): Flushing ebbbf218e8d25371b122092896fed576 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-27T21:34:54,800 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/ec1a339e460e4d4081fc6b7a98f72a46 is 50, key is test_row_0/A:col10/1732743292263/Put/seqid=0 2024-11-27T21:34:54,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741962_1138 (size=12301) 2024-11-27T21:34:54,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T21:34:55,204 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=602 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/ec1a339e460e4d4081fc6b7a98f72a46 2024-11-27T21:34:55,211 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/e3e6db71f4cb4bde8cb435c27d19e9a4 is 50, key is test_row_0/B:col10/1732743292263/Put/seqid=0 2024-11-27T21:34:55,212 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T21:34:55,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741963_1139 (size=12301) 2024-11-27T21:34:55,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T21:34:55,615 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=602 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/e3e6db71f4cb4bde8cb435c27d19e9a4 2024-11-27T21:34:55,622 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/bd836a74da244ee8b08c31af82d609a5 is 50, key is test_row_0/C:col10/1732743292263/Put/seqid=0 2024-11-27T21:34:55,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741964_1140 (size=12301) 2024-11-27T21:34:55,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T21:34:56,027 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=602 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/bd836a74da244ee8b08c31af82d609a5 2024-11-27T21:34:56,032 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/A/ec1a339e460e4d4081fc6b7a98f72a46 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/ec1a339e460e4d4081fc6b7a98f72a46 2024-11-27T21:34:56,036 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/ec1a339e460e4d4081fc6b7a98f72a46, entries=150, sequenceid=602, filesize=12.0 K 2024-11-27T21:34:56,037 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/B/e3e6db71f4cb4bde8cb435c27d19e9a4 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/e3e6db71f4cb4bde8cb435c27d19e9a4 2024-11-27T21:34:56,040 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/e3e6db71f4cb4bde8cb435c27d19e9a4, entries=150, sequenceid=602, filesize=12.0 K 2024-11-27T21:34:56,041 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/.tmp/C/bd836a74da244ee8b08c31af82d609a5 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/bd836a74da244ee8b08c31af82d609a5 2024-11-27T21:34:56,046 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/bd836a74da244ee8b08c31af82d609a5, entries=150, sequenceid=602, filesize=12.0 K 2024-11-27T21:34:56,047 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for ebbbf218e8d25371b122092896fed576 in 1252ms, sequenceid=602, compaction requested=true 2024-11-27T21:34:56,047 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/c3077b7e16ee482baf5e1d6b0b78321c, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/612917100cf9424a8c9e224ff7d205a6, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/c9d1d7a712e34571b41d75a7bbecbd63, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/004d1554d55e46cf95816377a9fbcc5a, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/8817a5a2e7e54398a2234500389c3b6a, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/64ea678994cf4529a5396c4066236b79, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/a87d9a4b53644c68815732fad1a4bd15, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/5a769eb8bd604ce482058c8f86e67f81, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/f127cd9d03db4d3f848e9e61531160b5, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/eae2c79801574a07916fcfac427d583b, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/806b189670194c978c0c97961f271295, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/ad08726af6c94411a101ece386e7966e, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/1a609947b9764a819e710d7b0b8fdcf0, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/bc47f9b9b6ea442cbae08d6909963fa3, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/ec70ef3627bb408ea5b767217d4c9745, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/84975eaa4d1243afa57a32213ff34942, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/b6c1428f7d0f40e5aa5a54d845bbb830, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/dc100c800938408fa91745bd32f90639, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/51dfcc094e9a4363ad69d5e429c4e333, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/1e4f3bccfe0e48baba3f049078627c37, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/ba6aa291d9e5420f95bbf773692e0652, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/9cc69270007944f6a5fe6507b3681787, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/28c40e72b0f9438d8c8e0392a93aa99c, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/89a1482c64d443978b1e850ca7a7e537, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/287c9052832a43c09459c5b178da4fa5, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/bf8f6c2f554f4043bda9c83d4336e03f, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/1e13b19d982a4407ba498b874b0929db, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/b645eec5b9fb46ddbe7f396a7b3ad4ec, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/192caad9dd5a4bfe8a9bd9868826569c, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/bade09d1cec64c30ab8ed239c09661ed, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/0d49f2abcf6d4baab5543e79008a6265, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/6aae2eb4a00b458cb4b020757fb45563, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/c9c786d2080d44e998e75dc73d451ba3, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/322ffc92df9a4c4ab6d6b361c469afb5, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/6e1a1c5705be458cb0ddf6d17c064ee6, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/15c3cdb202c74d968d35b5947e91d6d9, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/fc8af206cac24a6fb62a569308746f81, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/b4dd9786520a4fb89d17b66f6bab8067, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/b3d1a9e9ab1541beba2e2a1a175624da] to archive 2024-11-27T21:34:56,050 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T21:34:56,056 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/c3077b7e16ee482baf5e1d6b0b78321c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/c3077b7e16ee482baf5e1d6b0b78321c 2024-11-27T21:34:56,057 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/612917100cf9424a8c9e224ff7d205a6 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/612917100cf9424a8c9e224ff7d205a6 2024-11-27T21:34:56,058 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/c9d1d7a712e34571b41d75a7bbecbd63 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/c9d1d7a712e34571b41d75a7bbecbd63 2024-11-27T21:34:56,060 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/004d1554d55e46cf95816377a9fbcc5a to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/004d1554d55e46cf95816377a9fbcc5a 2024-11-27T21:34:56,061 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/8817a5a2e7e54398a2234500389c3b6a to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/8817a5a2e7e54398a2234500389c3b6a 2024-11-27T21:34:56,062 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/64ea678994cf4529a5396c4066236b79 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/64ea678994cf4529a5396c4066236b79 2024-11-27T21:34:56,063 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/a87d9a4b53644c68815732fad1a4bd15 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/a87d9a4b53644c68815732fad1a4bd15 2024-11-27T21:34:56,064 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/5a769eb8bd604ce482058c8f86e67f81 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/5a769eb8bd604ce482058c8f86e67f81 2024-11-27T21:34:56,065 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/f127cd9d03db4d3f848e9e61531160b5 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/f127cd9d03db4d3f848e9e61531160b5 2024-11-27T21:34:56,066 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/eae2c79801574a07916fcfac427d583b to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/eae2c79801574a07916fcfac427d583b 2024-11-27T21:34:56,067 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/806b189670194c978c0c97961f271295 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/806b189670194c978c0c97961f271295 2024-11-27T21:34:56,069 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/ad08726af6c94411a101ece386e7966e to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/ad08726af6c94411a101ece386e7966e 2024-11-27T21:34:56,070 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/1a609947b9764a819e710d7b0b8fdcf0 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/1a609947b9764a819e710d7b0b8fdcf0 2024-11-27T21:34:56,071 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/bc47f9b9b6ea442cbae08d6909963fa3 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/bc47f9b9b6ea442cbae08d6909963fa3 2024-11-27T21:34:56,072 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/ec70ef3627bb408ea5b767217d4c9745 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/ec70ef3627bb408ea5b767217d4c9745 2024-11-27T21:34:56,073 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/84975eaa4d1243afa57a32213ff34942 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/84975eaa4d1243afa57a32213ff34942 2024-11-27T21:34:56,075 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/b6c1428f7d0f40e5aa5a54d845bbb830 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/b6c1428f7d0f40e5aa5a54d845bbb830 2024-11-27T21:34:56,076 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/dc100c800938408fa91745bd32f90639 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/dc100c800938408fa91745bd32f90639 2024-11-27T21:34:56,077 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/51dfcc094e9a4363ad69d5e429c4e333 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/51dfcc094e9a4363ad69d5e429c4e333 2024-11-27T21:34:56,078 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/1e4f3bccfe0e48baba3f049078627c37 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/1e4f3bccfe0e48baba3f049078627c37 2024-11-27T21:34:56,079 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/ba6aa291d9e5420f95bbf773692e0652 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/ba6aa291d9e5420f95bbf773692e0652 2024-11-27T21:34:56,080 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/9cc69270007944f6a5fe6507b3681787 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/9cc69270007944f6a5fe6507b3681787 2024-11-27T21:34:56,082 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/28c40e72b0f9438d8c8e0392a93aa99c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/28c40e72b0f9438d8c8e0392a93aa99c 2024-11-27T21:34:56,083 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/89a1482c64d443978b1e850ca7a7e537 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/89a1482c64d443978b1e850ca7a7e537 2024-11-27T21:34:56,084 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/287c9052832a43c09459c5b178da4fa5 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/287c9052832a43c09459c5b178da4fa5 2024-11-27T21:34:56,085 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/bf8f6c2f554f4043bda9c83d4336e03f to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/bf8f6c2f554f4043bda9c83d4336e03f 2024-11-27T21:34:56,086 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/1e13b19d982a4407ba498b874b0929db to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/1e13b19d982a4407ba498b874b0929db 2024-11-27T21:34:56,088 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/b645eec5b9fb46ddbe7f396a7b3ad4ec to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/b645eec5b9fb46ddbe7f396a7b3ad4ec 2024-11-27T21:34:56,089 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/192caad9dd5a4bfe8a9bd9868826569c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/192caad9dd5a4bfe8a9bd9868826569c 2024-11-27T21:34:56,090 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/bade09d1cec64c30ab8ed239c09661ed to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/bade09d1cec64c30ab8ed239c09661ed 2024-11-27T21:34:56,091 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/0d49f2abcf6d4baab5543e79008a6265 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/0d49f2abcf6d4baab5543e79008a6265 2024-11-27T21:34:56,092 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/6aae2eb4a00b458cb4b020757fb45563 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/6aae2eb4a00b458cb4b020757fb45563 2024-11-27T21:34:56,093 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/c9c786d2080d44e998e75dc73d451ba3 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/c9c786d2080d44e998e75dc73d451ba3 2024-11-27T21:34:56,094 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/322ffc92df9a4c4ab6d6b361c469afb5 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/322ffc92df9a4c4ab6d6b361c469afb5 2024-11-27T21:34:56,096 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/6e1a1c5705be458cb0ddf6d17c064ee6 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/6e1a1c5705be458cb0ddf6d17c064ee6 2024-11-27T21:34:56,097 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/15c3cdb202c74d968d35b5947e91d6d9 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/15c3cdb202c74d968d35b5947e91d6d9 2024-11-27T21:34:56,098 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/fc8af206cac24a6fb62a569308746f81 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/fc8af206cac24a6fb62a569308746f81 2024-11-27T21:34:56,099 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/b4dd9786520a4fb89d17b66f6bab8067 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/b4dd9786520a4fb89d17b66f6bab8067 2024-11-27T21:34:56,100 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/b3d1a9e9ab1541beba2e2a1a175624da to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/b3d1a9e9ab1541beba2e2a1a175624da 2024-11-27T21:34:56,115 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/7bf3630f62bd49e6aa1d034d1c399c7c, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/fdf1aa591d674cf7a40fbc37e1fd5ce9, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/90cad7c46e874928b6829a87844615fa, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/f779a87105f4488eaef658088fe1264b, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/81cdaf13322849e1b17bd21bd1d9f142, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/81d3d073647d461c8c5b3c7a9a06230f, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/aeb3a246cddc40d0bcd902aab1e9ee9e, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/9f43213a8a9d474596add0e87a544242, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/f260d87f55eb43229f8f6083a591cf18, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/efe1a86a1a1e4ff4898b95a77bc214b1, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/6e88e6aea7184e489de554d3c51ff664, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/9c41c56cd09e4c1b8fe2f2df84579fea, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/4557ee9ad46240a7abf2d005696d0eb2, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/e71be646b259479480da4dcfaa74e3c5, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/9a619ac3871247a7b3663273901034d9, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/f46405dfac774be0a7d5d2d1571b9f1d, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/045af624626a4b0ca42b73be1510a7c9, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/20a07468e11e4ef8bf0a1615446a0dca, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/96a8c2a70fb74f5da971c1d12a661e98, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/3516d22fcb5d4e758a517537d1d3e934, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/62fa5f33b32b41e393b66d86ba261376, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/0cd3135f51c542f3bb7f54a33d80a2a0, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/a8b9a39bdeff4d96888389dcb2e2c3ec, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/b35e86ca12a842008367651522ef63ff, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/08bf262bd3354cbab381cebebb9383e8, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/90bea62db87847acaf2635dba524f8c2, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/4d807145557c41aa88020532881f6e0c, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/21badf024c25408cb484ee62eaa61569, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/28d9674c034041dfb58408ecad093124, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/d667b1fbf6b34c92a61749030ec94121, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/89d3d6af96b2444bbac6d6b75fa62b0f, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/864e0b875f7d4726aaabf76276024741, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/fb8d375f43c844da96580fb08cddb604, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/886e88fdbdb84cd5b1ad55975664aeac, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/b1a961c1743b45948578b98774ff1900, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/f898192e973a4318b1556db945e032eb, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/165b8c4eb3b7414eae6551cd99a87b48, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/11647a6c65f54a1d909157fc08977f61, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/ef68b765210d450fbeef23e1a2c5e15a] to archive 2024-11-27T21:34:56,116 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T21:34:56,118 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/7bf3630f62bd49e6aa1d034d1c399c7c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/7bf3630f62bd49e6aa1d034d1c399c7c 2024-11-27T21:34:56,119 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/fdf1aa591d674cf7a40fbc37e1fd5ce9 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/fdf1aa591d674cf7a40fbc37e1fd5ce9 2024-11-27T21:34:56,121 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/90cad7c46e874928b6829a87844615fa to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/90cad7c46e874928b6829a87844615fa 2024-11-27T21:34:56,122 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/f779a87105f4488eaef658088fe1264b to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/f779a87105f4488eaef658088fe1264b 2024-11-27T21:34:56,123 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/81cdaf13322849e1b17bd21bd1d9f142 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/81cdaf13322849e1b17bd21bd1d9f142 2024-11-27T21:34:56,124 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/81d3d073647d461c8c5b3c7a9a06230f to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/81d3d073647d461c8c5b3c7a9a06230f 2024-11-27T21:34:56,126 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/aeb3a246cddc40d0bcd902aab1e9ee9e to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/aeb3a246cddc40d0bcd902aab1e9ee9e 2024-11-27T21:34:56,127 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/9f43213a8a9d474596add0e87a544242 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/9f43213a8a9d474596add0e87a544242 2024-11-27T21:34:56,128 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/f260d87f55eb43229f8f6083a591cf18 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/f260d87f55eb43229f8f6083a591cf18 2024-11-27T21:34:56,129 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/efe1a86a1a1e4ff4898b95a77bc214b1 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/efe1a86a1a1e4ff4898b95a77bc214b1 2024-11-27T21:34:56,131 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/6e88e6aea7184e489de554d3c51ff664 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/6e88e6aea7184e489de554d3c51ff664 2024-11-27T21:34:56,132 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/9c41c56cd09e4c1b8fe2f2df84579fea to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/9c41c56cd09e4c1b8fe2f2df84579fea 2024-11-27T21:34:56,133 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/4557ee9ad46240a7abf2d005696d0eb2 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/4557ee9ad46240a7abf2d005696d0eb2 2024-11-27T21:34:56,135 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/e71be646b259479480da4dcfaa74e3c5 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/e71be646b259479480da4dcfaa74e3c5 2024-11-27T21:34:56,136 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/9a619ac3871247a7b3663273901034d9 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/9a619ac3871247a7b3663273901034d9 2024-11-27T21:34:56,137 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/f46405dfac774be0a7d5d2d1571b9f1d to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/f46405dfac774be0a7d5d2d1571b9f1d 2024-11-27T21:34:56,138 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/045af624626a4b0ca42b73be1510a7c9 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/045af624626a4b0ca42b73be1510a7c9 2024-11-27T21:34:56,140 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/20a07468e11e4ef8bf0a1615446a0dca to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/20a07468e11e4ef8bf0a1615446a0dca 2024-11-27T21:34:56,141 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/96a8c2a70fb74f5da971c1d12a661e98 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/96a8c2a70fb74f5da971c1d12a661e98 2024-11-27T21:34:56,142 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/3516d22fcb5d4e758a517537d1d3e934 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/3516d22fcb5d4e758a517537d1d3e934 2024-11-27T21:34:56,143 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/62fa5f33b32b41e393b66d86ba261376 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/62fa5f33b32b41e393b66d86ba261376 2024-11-27T21:34:56,144 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/0cd3135f51c542f3bb7f54a33d80a2a0 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/0cd3135f51c542f3bb7f54a33d80a2a0 2024-11-27T21:34:56,145 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/a8b9a39bdeff4d96888389dcb2e2c3ec to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/a8b9a39bdeff4d96888389dcb2e2c3ec 2024-11-27T21:34:56,147 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/b35e86ca12a842008367651522ef63ff to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/b35e86ca12a842008367651522ef63ff 2024-11-27T21:34:56,148 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/08bf262bd3354cbab381cebebb9383e8 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/08bf262bd3354cbab381cebebb9383e8 2024-11-27T21:34:56,149 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/90bea62db87847acaf2635dba524f8c2 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/90bea62db87847acaf2635dba524f8c2 2024-11-27T21:34:56,150 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/4d807145557c41aa88020532881f6e0c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/4d807145557c41aa88020532881f6e0c 2024-11-27T21:34:56,151 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/21badf024c25408cb484ee62eaa61569 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/21badf024c25408cb484ee62eaa61569 2024-11-27T21:34:56,152 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/28d9674c034041dfb58408ecad093124 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/28d9674c034041dfb58408ecad093124 2024-11-27T21:34:56,153 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/d667b1fbf6b34c92a61749030ec94121 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/d667b1fbf6b34c92a61749030ec94121 2024-11-27T21:34:56,154 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/89d3d6af96b2444bbac6d6b75fa62b0f to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/89d3d6af96b2444bbac6d6b75fa62b0f 2024-11-27T21:34:56,155 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/864e0b875f7d4726aaabf76276024741 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/864e0b875f7d4726aaabf76276024741 2024-11-27T21:34:56,156 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/fb8d375f43c844da96580fb08cddb604 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/fb8d375f43c844da96580fb08cddb604 2024-11-27T21:34:56,157 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/886e88fdbdb84cd5b1ad55975664aeac to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/886e88fdbdb84cd5b1ad55975664aeac 2024-11-27T21:34:56,158 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/b1a961c1743b45948578b98774ff1900 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/b1a961c1743b45948578b98774ff1900 2024-11-27T21:34:56,159 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/f898192e973a4318b1556db945e032eb to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/f898192e973a4318b1556db945e032eb 2024-11-27T21:34:56,160 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/165b8c4eb3b7414eae6551cd99a87b48 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/165b8c4eb3b7414eae6551cd99a87b48 2024-11-27T21:34:56,161 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/11647a6c65f54a1d909157fc08977f61 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/11647a6c65f54a1d909157fc08977f61 2024-11-27T21:34:56,162 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/ef68b765210d450fbeef23e1a2c5e15a to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/ef68b765210d450fbeef23e1a2c5e15a 2024-11-27T21:34:56,164 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/b3a2393c3b6f41f6a902cf18a68a0d44, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/c47faff8a40741549bb282403c543b15, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/8587652b7d5c49e8a3991f4ba9473aad, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/59ac1624539f457fb3bc71b3cd465c81, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/10751b3d4b944776b7cea3c716027319, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/986a22de11a24b168a30ea2c66448fa5, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/739ed682efff463ba1d40d94e12cc31e, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/d334176494d649dfa1fc4cc31b9ab872, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/ecc6f2840bd94670809c7b51966d109a, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/37faccf655c842b9847968f35cd5ecc4, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/bf60ddaa82bb40a6991f517008d5f981, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/cd2d5a09148547eab02e01f305a5d0fc, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/aa73496fbadf468387fe5db4d08574d0, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/287361b1b5cf4d1197963bb16960d751, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/b8c0574b668e4479bc166cb5800c8058, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/3f2a4ed6a9174bc98ad4a3757c4eb40a, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/3785734deb9b49d3b20eddf7d66f0cf3, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/a35cc849614b433c98dd01663389c1cf, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/0b9d68778ad248f79a6710a040d8ebb1, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/6a466c7264004a3cbf8ed003e28dfa64, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/3c3b35bc105b4cafbf1c44f94ea8908e, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/0ea9fab9cd654ba798b6b3f89e0fa719, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/c4afcc83fdeb491a8acaf1335a200e2d, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/e28cd0752e894c0a8bfbaceba0e4aead, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/fe9a296d8fc443c8a259feadc02391f8, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/691aa8864c8c40ed8ff7c9f416077b67, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/2b38bcf7829f4bd6a1dbee4b1408135a, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/51c8051f8781422fa3f3e7fa8d4db854, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/974899bfd3294a46ace9eb40c7c58472, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/60527239776b43c8894658bb85bc2a4a, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/7dea71ca2b2b4c40a417055baadb1bb6, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/48dced978ad14ed5bc0340ad6d50055d, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/316625f62c7141e3a71292232b41f72e, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/7e7f01f7a0e0427bb076455cb5b27139, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/0bf415e6893a44fdba6054db257e6b28, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/54e059546a0e4e329c821e0efd59ff29, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/329dfe9276d24538bc65472253c7e400, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/9b625497959b4b3b8ed7fd101ad3dd1f] to archive 2024-11-27T21:34:56,165 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T21:34:56,166 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/b3a2393c3b6f41f6a902cf18a68a0d44 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/b3a2393c3b6f41f6a902cf18a68a0d44 2024-11-27T21:34:56,167 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/c47faff8a40741549bb282403c543b15 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/c47faff8a40741549bb282403c543b15 2024-11-27T21:34:56,168 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/8587652b7d5c49e8a3991f4ba9473aad to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/8587652b7d5c49e8a3991f4ba9473aad 2024-11-27T21:34:56,169 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/59ac1624539f457fb3bc71b3cd465c81 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/59ac1624539f457fb3bc71b3cd465c81 2024-11-27T21:34:56,171 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/10751b3d4b944776b7cea3c716027319 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/10751b3d4b944776b7cea3c716027319 2024-11-27T21:34:56,171 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/986a22de11a24b168a30ea2c66448fa5 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/986a22de11a24b168a30ea2c66448fa5 2024-11-27T21:34:56,173 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/739ed682efff463ba1d40d94e12cc31e to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/739ed682efff463ba1d40d94e12cc31e 2024-11-27T21:34:56,174 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/d334176494d649dfa1fc4cc31b9ab872 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/d334176494d649dfa1fc4cc31b9ab872 2024-11-27T21:34:56,175 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/ecc6f2840bd94670809c7b51966d109a to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/ecc6f2840bd94670809c7b51966d109a 2024-11-27T21:34:56,176 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/37faccf655c842b9847968f35cd5ecc4 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/37faccf655c842b9847968f35cd5ecc4 2024-11-27T21:34:56,177 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/bf60ddaa82bb40a6991f517008d5f981 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/bf60ddaa82bb40a6991f517008d5f981 2024-11-27T21:34:56,178 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/cd2d5a09148547eab02e01f305a5d0fc to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/cd2d5a09148547eab02e01f305a5d0fc 2024-11-27T21:34:56,179 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/aa73496fbadf468387fe5db4d08574d0 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/aa73496fbadf468387fe5db4d08574d0 2024-11-27T21:34:56,180 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/287361b1b5cf4d1197963bb16960d751 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/287361b1b5cf4d1197963bb16960d751 2024-11-27T21:34:56,181 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/b8c0574b668e4479bc166cb5800c8058 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/b8c0574b668e4479bc166cb5800c8058 2024-11-27T21:34:56,182 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/3f2a4ed6a9174bc98ad4a3757c4eb40a to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/3f2a4ed6a9174bc98ad4a3757c4eb40a 2024-11-27T21:34:56,183 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/3785734deb9b49d3b20eddf7d66f0cf3 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/3785734deb9b49d3b20eddf7d66f0cf3 2024-11-27T21:34:56,184 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/a35cc849614b433c98dd01663389c1cf to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/a35cc849614b433c98dd01663389c1cf 2024-11-27T21:34:56,185 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/0b9d68778ad248f79a6710a040d8ebb1 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/0b9d68778ad248f79a6710a040d8ebb1 2024-11-27T21:34:56,186 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/6a466c7264004a3cbf8ed003e28dfa64 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/6a466c7264004a3cbf8ed003e28dfa64 2024-11-27T21:34:56,187 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/3c3b35bc105b4cafbf1c44f94ea8908e to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/3c3b35bc105b4cafbf1c44f94ea8908e 2024-11-27T21:34:56,188 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/0ea9fab9cd654ba798b6b3f89e0fa719 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/0ea9fab9cd654ba798b6b3f89e0fa719 2024-11-27T21:34:56,189 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/c4afcc83fdeb491a8acaf1335a200e2d to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/c4afcc83fdeb491a8acaf1335a200e2d 2024-11-27T21:34:56,190 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/e28cd0752e894c0a8bfbaceba0e4aead to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/e28cd0752e894c0a8bfbaceba0e4aead 2024-11-27T21:34:56,191 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/fe9a296d8fc443c8a259feadc02391f8 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/fe9a296d8fc443c8a259feadc02391f8 2024-11-27T21:34:56,192 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/691aa8864c8c40ed8ff7c9f416077b67 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/691aa8864c8c40ed8ff7c9f416077b67 2024-11-27T21:34:56,193 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/2b38bcf7829f4bd6a1dbee4b1408135a to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/2b38bcf7829f4bd6a1dbee4b1408135a 2024-11-27T21:34:56,194 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/51c8051f8781422fa3f3e7fa8d4db854 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/51c8051f8781422fa3f3e7fa8d4db854 2024-11-27T21:34:56,195 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/974899bfd3294a46ace9eb40c7c58472 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/974899bfd3294a46ace9eb40c7c58472 2024-11-27T21:34:56,196 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/60527239776b43c8894658bb85bc2a4a to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/60527239776b43c8894658bb85bc2a4a 2024-11-27T21:34:56,197 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/7dea71ca2b2b4c40a417055baadb1bb6 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/7dea71ca2b2b4c40a417055baadb1bb6 2024-11-27T21:34:56,198 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/48dced978ad14ed5bc0340ad6d50055d to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/48dced978ad14ed5bc0340ad6d50055d 2024-11-27T21:34:56,199 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/316625f62c7141e3a71292232b41f72e to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/316625f62c7141e3a71292232b41f72e 2024-11-27T21:34:56,200 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/7e7f01f7a0e0427bb076455cb5b27139 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/7e7f01f7a0e0427bb076455cb5b27139 2024-11-27T21:34:56,201 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/0bf415e6893a44fdba6054db257e6b28 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/0bf415e6893a44fdba6054db257e6b28 2024-11-27T21:34:56,203 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/54e059546a0e4e329c821e0efd59ff29 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/54e059546a0e4e329c821e0efd59ff29 2024-11-27T21:34:56,204 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/329dfe9276d24538bc65472253c7e400 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/329dfe9276d24538bc65472253c7e400 2024-11-27T21:34:56,205 DEBUG [StoreCloser-TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/9b625497959b4b3b8ed7fd101ad3dd1f to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/9b625497959b4b3b8ed7fd101ad3dd1f 2024-11-27T21:34:56,209 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/recovered.edits/605.seqid, newMaxSeqId=605, maxSeqId=1 2024-11-27T21:34:56,210 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. 2024-11-27T21:34:56,210 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for ebbbf218e8d25371b122092896fed576: Waiting for close lock at 1732743294795Running coprocessor pre-close hooks at 1732743294795Disabling compacts and flushes for region at 1732743294795Disabling writes for close at 1732743294795Obtaining lock to block concurrent updates at 1732743294795Preparing flush snapshotting stores in ebbbf218e8d25371b122092896fed576 at 1732743294795Finished memstore snapshotting TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732743294796 (+1 ms)Flushing stores of TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576. at 1732743294796Flushing ebbbf218e8d25371b122092896fed576/A: creating writer at 1732743294796Flushing ebbbf218e8d25371b122092896fed576/A: appending metadata at 1732743294799 (+3 ms)Flushing ebbbf218e8d25371b122092896fed576/A: closing flushed file at 1732743294799Flushing ebbbf218e8d25371b122092896fed576/B: creating writer at 1732743295208 (+409 ms)Flushing ebbbf218e8d25371b122092896fed576/B: appending metadata at 1732743295210 (+2 ms)Flushing ebbbf218e8d25371b122092896fed576/B: closing flushed file at 1732743295210Flushing ebbbf218e8d25371b122092896fed576/C: creating writer at 1732743295619 (+409 ms)Flushing ebbbf218e8d25371b122092896fed576/C: appending metadata at 1732743295622 (+3 ms)Flushing ebbbf218e8d25371b122092896fed576/C: closing flushed file at 1732743295622Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b628297: reopening flushed file at 1732743296031 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6175f310: reopening flushed file at 1732743296036 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@359a3251: reopening flushed file at 1732743296041 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for ebbbf218e8d25371b122092896fed576 in 1252ms, sequenceid=602, compaction requested=true at 1732743296047 (+6 ms)Writing region close event to WAL at 1732743296206 (+159 ms)Running coprocessor post-close hooks at 1732743296210 (+4 ms)Closed at 1732743296210 2024-11-27T21:34:56,213 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed ebbbf218e8d25371b122092896fed576 2024-11-27T21:34:56,213 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=ebbbf218e8d25371b122092896fed576, regionState=CLOSED 2024-11-27T21:34:56,215 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 because future has completed 2024-11-27T21:34:56,217 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-27T21:34:56,217 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure ebbbf218e8d25371b122092896fed576, server=4585c39bed03,39097,1732743267048 in 1.5780 sec 2024-11-27T21:34:56,219 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-27T21:34:56,219 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ebbbf218e8d25371b122092896fed576, UNASSIGN in 1.5840 sec 2024-11-27T21:34:56,221 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-27T21:34:56,221 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5900 sec 2024-11-27T21:34:56,222 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732743296222"}]},"ts":"1732743296222"} 2024-11-27T21:34:56,223 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-27T21:34:56,223 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-27T21:34:56,225 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6040 sec 2024-11-27T21:34:56,698 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-27T21:34:56,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T21:34:56,783 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:34:56,787 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-27T21:34:56,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:34:56,792 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:34:56,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-27T21:34:56,793 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:34:56,797 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576 2024-11-27T21:34:56,801 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A, FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B, FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C, FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/recovered.edits] 2024-11-27T21:34:56,804 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/122e4c88acdb45689fd577f427cab249 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/122e4c88acdb45689fd577f427cab249 2024-11-27T21:34:56,805 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/6cd738af8b134d889f4c141e26587a25 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/6cd738af8b134d889f4c141e26587a25 2024-11-27T21:34:56,807 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/b93b06a5362545ffa3041ab67cded9f9 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/b93b06a5362545ffa3041ab67cded9f9 2024-11-27T21:34:56,808 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/ec1a339e460e4d4081fc6b7a98f72a46 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/A/ec1a339e460e4d4081fc6b7a98f72a46 2024-11-27T21:34:56,811 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/2b4c6ecba1ea447eb4d67d351db7e0d0 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/2b4c6ecba1ea447eb4d67d351db7e0d0 2024-11-27T21:34:56,813 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/50b7315ba3744a20bafcdbe7b8ac9c59 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/50b7315ba3744a20bafcdbe7b8ac9c59 2024-11-27T21:34:56,814 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/dfc82f120cf04a82864dc1d42d00d6f5 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/dfc82f120cf04a82864dc1d42d00d6f5 2024-11-27T21:34:56,815 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/e3e6db71f4cb4bde8cb435c27d19e9a4 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/B/e3e6db71f4cb4bde8cb435c27d19e9a4 2024-11-27T21:34:56,818 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/188d731cb47d4910a630993c2fdab611 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/188d731cb47d4910a630993c2fdab611 2024-11-27T21:34:56,820 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/364edfb94d404490bd84db0f5a80f9c9 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/364edfb94d404490bd84db0f5a80f9c9 2024-11-27T21:34:56,821 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/aeb21ddae35c432a97215478c9a2f90c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/aeb21ddae35c432a97215478c9a2f90c 2024-11-27T21:34:56,822 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/bd836a74da244ee8b08c31af82d609a5 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/C/bd836a74da244ee8b08c31af82d609a5 2024-11-27T21:34:56,825 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/recovered.edits/605.seqid to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576/recovered.edits/605.seqid 2024-11-27T21:34:56,826 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/ebbbf218e8d25371b122092896fed576 2024-11-27T21:34:56,826 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-27T21:34:56,831 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:34:56,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-27T21:34:56,840 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-27T21:34:56,869 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-27T21:34:56,870 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:34:56,870 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-27T21:34:56,871 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732743296870"}]},"ts":"9223372036854775807"} 2024-11-27T21:34:56,874 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-27T21:34:56,874 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ebbbf218e8d25371b122092896fed576, NAME => 'TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576.', STARTKEY => '', ENDKEY => ''}] 2024-11-27T21:34:56,875 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-27T21:34:56,875 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732743296875"}]},"ts":"9223372036854775807"} 2024-11-27T21:34:56,877 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-27T21:34:56,878 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:34:56,880 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 91 msec 2024-11-27T21:34:56,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-27T21:34:56,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-27T21:34:56,904 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:34:56,920 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=239 (was 211) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;4585c39bed03:39097-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_941021170_22 at /127.0.0.1:48462 [Waiting for operation #101] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_941021170_22 at /127.0.0.1:57280 [Waiting for operation #359] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1911604878_22 at /127.0.0.1:33876 [Waiting for operation #343] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/4585c39bed03:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=341 (was 265) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8557 (was 9233) 2024-11-27T21:34:56,932 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=239, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=341, ProcessCount=11, AvailableMemoryMB=8554 2024-11-27T21:34:56,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T21:34:56,934 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T21:34:56,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-27T21:34:56,937 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T21:34:56,937 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:34:56,937 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-11-27T21:34:56,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-27T21:34:56,938 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T21:34:56,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741965_1141 (size=959) 2024-11-27T21:34:57,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-27T21:34:57,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-27T21:34:57,347 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d 2024-11-27T21:34:57,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741966_1142 (size=53) 2024-11-27T21:34:57,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-27T21:34:57,753 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T21:34:57,753 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 4c65e64f21dac3e5db22a4c47b796905, disabling compactions & flushes 2024-11-27T21:34:57,753 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:34:57,753 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:34:57,753 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. after waiting 0 ms 2024-11-27T21:34:57,753 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:34:57,753 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:34:57,753 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4c65e64f21dac3e5db22a4c47b796905: Waiting for close lock at 1732743297753Disabling compacts and flushes for region at 1732743297753Disabling writes for close at 1732743297753Writing region close event to WAL at 1732743297753Closed at 1732743297753 2024-11-27T21:34:57,754 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T21:34:57,754 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732743297754"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732743297754"}]},"ts":"1732743297754"} 2024-11-27T21:34:57,756 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T21:34:57,757 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T21:34:57,757 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732743297757"}]},"ts":"1732743297757"} 2024-11-27T21:34:57,759 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-27T21:34:57,759 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c65e64f21dac3e5db22a4c47b796905, ASSIGN}] 2024-11-27T21:34:57,760 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c65e64f21dac3e5db22a4c47b796905, ASSIGN 2024-11-27T21:34:57,761 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c65e64f21dac3e5db22a4c47b796905, ASSIGN; state=OFFLINE, location=4585c39bed03,39097,1732743267048; forceNewPlan=false, retain=false 2024-11-27T21:34:57,912 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=4c65e64f21dac3e5db22a4c47b796905, regionState=OPENING, regionLocation=4585c39bed03,39097,1732743267048 2024-11-27T21:34:57,914 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c65e64f21dac3e5db22a4c47b796905, ASSIGN because future has completed 2024-11-27T21:34:57,914 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048}] 2024-11-27T21:34:58,070 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:34:58,070 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} 2024-11-27T21:34:58,070 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:34:58,070 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T21:34:58,070 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:34:58,070 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:34:58,072 INFO [StoreOpener-4c65e64f21dac3e5db22a4c47b796905-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:34:58,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-27T21:34:58,074 INFO [StoreOpener-4c65e64f21dac3e5db22a4c47b796905-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c65e64f21dac3e5db22a4c47b796905 columnFamilyName A 2024-11-27T21:34:58,074 DEBUG [StoreOpener-4c65e64f21dac3e5db22a4c47b796905-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:34:58,074 INFO [StoreOpener-4c65e64f21dac3e5db22a4c47b796905-1 {}] regionserver.HStore(327): Store=4c65e64f21dac3e5db22a4c47b796905/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:34:58,074 INFO [StoreOpener-4c65e64f21dac3e5db22a4c47b796905-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:34:58,075 INFO [StoreOpener-4c65e64f21dac3e5db22a4c47b796905-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c65e64f21dac3e5db22a4c47b796905 columnFamilyName B 2024-11-27T21:34:58,075 DEBUG [StoreOpener-4c65e64f21dac3e5db22a4c47b796905-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:34:58,076 INFO [StoreOpener-4c65e64f21dac3e5db22a4c47b796905-1 {}] regionserver.HStore(327): Store=4c65e64f21dac3e5db22a4c47b796905/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:34:58,076 INFO [StoreOpener-4c65e64f21dac3e5db22a4c47b796905-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:34:58,077 INFO [StoreOpener-4c65e64f21dac3e5db22a4c47b796905-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c65e64f21dac3e5db22a4c47b796905 columnFamilyName C 2024-11-27T21:34:58,077 DEBUG [StoreOpener-4c65e64f21dac3e5db22a4c47b796905-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:34:58,077 INFO [StoreOpener-4c65e64f21dac3e5db22a4c47b796905-1 {}] regionserver.HStore(327): Store=4c65e64f21dac3e5db22a4c47b796905/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:34:58,077 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:34:58,078 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:34:58,078 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:34:58,079 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:34:58,079 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:34:58,080 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T21:34:58,081 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:34:58,082 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T21:34:58,083 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 4c65e64f21dac3e5db22a4c47b796905; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66901032, jitterRate=-0.003096938133239746}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T21:34:58,083 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:34:58,084 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 4c65e64f21dac3e5db22a4c47b796905: Running coprocessor pre-open hook at 1732743298071Writing region info on filesystem at 1732743298071Initializing all the Stores at 1732743298071Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732743298071Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732743298071Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732743298071Cleaning up temporary data from old regions at 1732743298079 (+8 ms)Running coprocessor post-open hooks at 1732743298083 (+4 ms)Region opened successfully at 1732743298084 (+1 ms) 2024-11-27T21:34:58,084 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., pid=32, masterSystemTime=1732743298066 2024-11-27T21:34:58,086 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:34:58,086 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:34:58,087 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=4c65e64f21dac3e5db22a4c47b796905, regionState=OPEN, openSeqNum=2, regionLocation=4585c39bed03,39097,1732743267048 2024-11-27T21:34:58,088 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 because future has completed 2024-11-27T21:34:58,091 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-27T21:34:58,091 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure 4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 in 175 msec 2024-11-27T21:34:58,093 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-27T21:34:58,093 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c65e64f21dac3e5db22a4c47b796905, ASSIGN in 332 msec 2024-11-27T21:34:58,093 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T21:34:58,093 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732743298093"}]},"ts":"1732743298093"} 2024-11-27T21:34:58,095 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-27T21:34:58,096 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T21:34:58,097 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1610 sec 2024-11-27T21:34:59,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-27T21:34:59,083 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:34:59,085 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42e30321, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:34:59,085 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:34:59,085 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:34:59,087 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:34:59,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:34:59,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:34:59,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@733d2566, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:34:59,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:34:59,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:34:59,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:34:59,089 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39904, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:34:59,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@384bbb57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:34:59,091 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:34:59,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:34:59,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:34:59,093 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:34:59,095 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T21:34:59,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 4585c39bed03,34433,1732743266181 2024-11-27T21:34:59,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@696adb49 2024-11-27T21:34:59,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T21:34:59,097 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T21:34:59,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T21:34:59,102 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T21:34:59,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-27T21:34:59,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741967_1143 (size=995) 2024-11-27T21:34:59,523 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-27T21:34:59,524 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-27T21:34:59,527 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T21:34:59,533 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c65e64f21dac3e5db22a4c47b796905, REOPEN/MOVE}] 2024-11-27T21:34:59,534 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c65e64f21dac3e5db22a4c47b796905, REOPEN/MOVE 2024-11-27T21:34:59,535 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=4c65e64f21dac3e5db22a4c47b796905, regionState=CLOSING, regionLocation=4585c39bed03,39097,1732743267048 2024-11-27T21:34:59,536 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c65e64f21dac3e5db22a4c47b796905, REOPEN/MOVE because future has completed 2024-11-27T21:34:59,537 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T21:34:59,537 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048}] 2024-11-27T21:34:59,689 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close 4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:34:59,689 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T21:34:59,689 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing 4c65e64f21dac3e5db22a4c47b796905, disabling compactions & flushes 2024-11-27T21:34:59,689 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:34:59,689 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:34:59,689 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. after waiting 0 ms 2024-11-27T21:34:59,689 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:34:59,693 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T21:34:59,694 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:34:59,694 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for 4c65e64f21dac3e5db22a4c47b796905: Waiting for close lock at 1732743299689Running coprocessor pre-close hooks at 1732743299689Disabling compacts and flushes for region at 1732743299689Disabling writes for close at 1732743299689Writing region close event to WAL at 1732743299690 (+1 ms)Running coprocessor post-close hooks at 1732743299694 (+4 ms)Closed at 1732743299694 2024-11-27T21:34:59,694 WARN [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: 4c65e64f21dac3e5db22a4c47b796905 to self. 2024-11-27T21:34:59,696 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed 4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:34:59,696 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=4c65e64f21dac3e5db22a4c47b796905, regionState=CLOSED 2024-11-27T21:34:59,698 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 because future has completed 2024-11-27T21:34:59,700 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-27T21:34:59,700 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 in 162 msec 2024-11-27T21:34:59,701 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c65e64f21dac3e5db22a4c47b796905, REOPEN/MOVE; state=CLOSED, location=4585c39bed03,39097,1732743267048; forceNewPlan=false, retain=true 2024-11-27T21:34:59,851 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=4c65e64f21dac3e5db22a4c47b796905, regionState=OPENING, regionLocation=4585c39bed03,39097,1732743267048 2024-11-27T21:34:59,853 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c65e64f21dac3e5db22a4c47b796905, REOPEN/MOVE because future has completed 2024-11-27T21:34:59,854 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048}] 2024-11-27T21:35:00,009 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:00,009 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} 2024-11-27T21:35:00,009 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:00,009 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T21:35:00,009 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:00,009 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:00,013 INFO [StoreOpener-4c65e64f21dac3e5db22a4c47b796905-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:00,019 INFO [StoreOpener-4c65e64f21dac3e5db22a4c47b796905-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c65e64f21dac3e5db22a4c47b796905 columnFamilyName A 2024-11-27T21:35:00,021 DEBUG [StoreOpener-4c65e64f21dac3e5db22a4c47b796905-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:35:00,021 INFO [StoreOpener-4c65e64f21dac3e5db22a4c47b796905-1 {}] regionserver.HStore(327): Store=4c65e64f21dac3e5db22a4c47b796905/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:35:00,022 INFO [StoreOpener-4c65e64f21dac3e5db22a4c47b796905-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:00,022 INFO [StoreOpener-4c65e64f21dac3e5db22a4c47b796905-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c65e64f21dac3e5db22a4c47b796905 columnFamilyName B 2024-11-27T21:35:00,022 DEBUG [StoreOpener-4c65e64f21dac3e5db22a4c47b796905-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:35:00,023 INFO [StoreOpener-4c65e64f21dac3e5db22a4c47b796905-1 {}] regionserver.HStore(327): Store=4c65e64f21dac3e5db22a4c47b796905/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:35:00,023 INFO [StoreOpener-4c65e64f21dac3e5db22a4c47b796905-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:00,023 INFO [StoreOpener-4c65e64f21dac3e5db22a4c47b796905-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c65e64f21dac3e5db22a4c47b796905 columnFamilyName C 2024-11-27T21:35:00,023 DEBUG [StoreOpener-4c65e64f21dac3e5db22a4c47b796905-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:35:00,024 INFO [StoreOpener-4c65e64f21dac3e5db22a4c47b796905-1 {}] regionserver.HStore(327): Store=4c65e64f21dac3e5db22a4c47b796905/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:35:00,024 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:00,025 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:00,026 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:00,026 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:00,027 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:00,027 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T21:35:00,028 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:00,029 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 4c65e64f21dac3e5db22a4c47b796905; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69912882, jitterRate=0.04178312420845032}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T21:35:00,029 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:00,030 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 4c65e64f21dac3e5db22a4c47b796905: Running coprocessor pre-open hook at 1732743300009Writing region info on filesystem at 1732743300009Initializing all the Stores at 1732743300010 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732743300010Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732743300010Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732743300010Cleaning up temporary data from old regions at 1732743300027 (+17 ms)Running coprocessor post-open hooks at 1732743300029 (+2 ms)Region opened successfully at 1732743300030 (+1 ms) 2024-11-27T21:35:00,031 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., pid=37, masterSystemTime=1732743300006 2024-11-27T21:35:00,033 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:00,033 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:00,034 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=4c65e64f21dac3e5db22a4c47b796905, regionState=OPEN, openSeqNum=5, regionLocation=4585c39bed03,39097,1732743267048 2024-11-27T21:35:00,035 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 because future has completed 2024-11-27T21:35:00,038 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-11-27T21:35:00,038 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure 4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 in 182 msec 2024-11-27T21:35:00,040 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-27T21:35:00,040 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c65e64f21dac3e5db22a4c47b796905, REOPEN/MOVE in 505 msec 2024-11-27T21:35:00,042 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-27T21:35:00,043 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 514 msec 2024-11-27T21:35:00,046 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 939 msec 2024-11-27T21:35:00,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-27T21:35:00,048 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:35:00,049 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f9c3dcb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:00,049 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:35:00,049 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:35:00,050 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:35:00,050 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:35:00,050 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:35:00,050 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1be561f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:00,050 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:35:00,051 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:35:00,051 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:00,052 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39940, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:35:00,052 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4702c409, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:00,053 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dde8260, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:00,053 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:35:00,053 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:35:00,054 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:35:00,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:35:00,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:35:00,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@393389fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:00,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:35:00,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:35:00,055 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:00,056 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39956, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:35:00,056 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c43b88b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:00,057 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ab5c81d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:00,057 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:35:00,057 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:35:00,058 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:35:00,058 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:35:00,058 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:35:00,058 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52354a98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:00,058 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:35:00,059 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:35:00,059 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:00,060 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39972, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:35:00,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@700d8b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:00,061 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ea1a430, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:00,061 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:35:00,061 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:35:00,062 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:35:00,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:35:00,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:35:00,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@347ed380, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:00,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:35:00,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:35:00,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:00,064 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39996, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:35:00,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@efb48e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:00,065 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fbff363, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:00,065 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:35:00,066 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:35:00,066 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:35:00,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:35:00,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:35:00,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4227ccb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:00,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:35:00,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:35:00,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:00,068 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:35:00,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c4021c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:00,069 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@292bddff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:00,069 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:35:00,069 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:35:00,070 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:35:00,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:35:00,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:35:00,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69f1359e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:00,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:35:00,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:35:00,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:00,071 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40046, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:35:00,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@495919e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:00,072 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21a14e5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:00,072 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:35:00,072 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:35:00,073 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:35:00,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:35:00,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:35:00,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11ec6a05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:00,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:35:00,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:35:00,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:00,074 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40068, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:35:00,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ee33b11, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:00,075 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f17c234, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:00,075 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:35:00,076 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:35:00,076 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:35:00,076 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:35:00,076 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:35:00,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c213e11, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:00,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:35:00,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:35:00,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:00,077 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40094, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:35:00,078 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@642de49a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:00,079 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@197f7df3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:00,079 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:35:00,079 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:35:00,079 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:35:00,080 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:35:00,080 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:35:00,080 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ed4a045, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:00,080 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:35:00,080 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:35:00,080 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:00,081 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40110, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:35:00,081 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@228ba1de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:00,082 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:35:00,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-27T21:35:00,084 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:35:00,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-27T21:35:00,085 DEBUG [Thread-720 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:35:00,085 DEBUG [Thread-718 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:35:00,085 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:35:00,085 DEBUG [Thread-709 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:35:00,085 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:35:00,085 DEBUG [Thread-703 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:35:00,086 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:35:00,086 DEBUG [Thread-714 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:35:00,086 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:35:00,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:35:00,086 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:35:00,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:35:00,087 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:35:00,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:35:00,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:35:00,088 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41656, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:35:00,088 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41658, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:35:00,088 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41664, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:35:00,089 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41676, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:35:00,090 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:35:00,090 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:35:00,091 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41682, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:35:00,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5] 2024-11-27T21:35:00,092 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5] 2024-11-27T21:35:00,093 DEBUG [Thread-716 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:35:00,093 DEBUG [Thread-705 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:35:00,093 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:35:00,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:35:00,094 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:35:00,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5] 2024-11-27T21:35:00,094 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:35:00,095 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:35:00,095 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41692, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:35:00,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5] 2024-11-27T21:35:00,096 DEBUG [Thread-707 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:35:00,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:35:00,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:35:00,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5] 2024-11-27T21:35:00,099 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41708, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:35:00,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5] 2024-11-27T21:35:00,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5] 2024-11-27T21:35:00,103 DEBUG [Thread-711 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:35:00,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5] 2024-11-27T21:35:00,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:35:00,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:35:00,113 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:35:00,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5] 2024-11-27T21:35:00,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on 4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:00,120 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:35:00,120 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c65e64f21dac3e5db22a4c47b796905 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T21:35:00,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:00,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743360171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:00,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411270e24740406b547b2adcb3719345252b3_4c65e64f21dac3e5db22a4c47b796905 is 50, key is test_row_0/A:col10/1732743300114/Put/seqid=0 2024-11-27T21:35:00,182 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,182 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,182 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:00,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:00,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743360171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:00,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:00,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:00,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743360172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:00,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:00,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:00,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743360174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:00,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:00,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:00,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743360174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:00,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:00,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-27T21:35:00,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741968_1144 (size=19474) 2024-11-27T21:35:00,207 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:35:00,213 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411270e24740406b547b2adcb3719345252b3_4c65e64f21dac3e5db22a4c47b796905 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411270e24740406b547b2adcb3719345252b3_4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:00,214 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/4ecf5c5ffdbe49d59968dcd0667d1087, store: [table=TestAcidGuarantees family=A region=4c65e64f21dac3e5db22a4c47b796905] 2024-11-27T21:35:00,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/4ecf5c5ffdbe49d59968dcd0667d1087 is 175, key is test_row_0/A:col10/1732743300114/Put/seqid=0 2024-11-27T21:35:00,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-27T21:35:00,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:00,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. as already flushing 2024-11-27T21:35:00,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:00,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:00,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:00,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741969_1145 (size=56729) 2024-11-27T21:35:00,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:00,248 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/4ecf5c5ffdbe49d59968dcd0667d1087 2024-11-27T21:35:00,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/6d0e9bedae814a02a672fec49697df7c is 50, key is test_row_0/B:col10/1732743300114/Put/seqid=0 2024-11-27T21:35:00,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741970_1146 (size=9657) 2024-11-27T21:35:00,292 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/6d0e9bedae814a02a672fec49697df7c 2024-11-27T21:35:00,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:00,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743360294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:00,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:00,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743360294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:00,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:00,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743360295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:00,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:00,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:00,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743360295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:00,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:00,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:00,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:00,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:00,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743360295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:00,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:00,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/e8f1a0f4882e4ffba03bb678822007df is 50, key is test_row_0/C:col10/1732743300114/Put/seqid=0 2024-11-27T21:35:00,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741971_1147 (size=9657) 2024-11-27T21:35:00,331 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/e8f1a0f4882e4ffba03bb678822007df 2024-11-27T21:35:00,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/4ecf5c5ffdbe49d59968dcd0667d1087 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/4ecf5c5ffdbe49d59968dcd0667d1087 2024-11-27T21:35:00,346 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/4ecf5c5ffdbe49d59968dcd0667d1087, entries=300, sequenceid=16, filesize=55.4 K 2024-11-27T21:35:00,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/6d0e9bedae814a02a672fec49697df7c as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/6d0e9bedae814a02a672fec49697df7c 2024-11-27T21:35:00,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/6d0e9bedae814a02a672fec49697df7c, entries=100, sequenceid=16, filesize=9.4 K 2024-11-27T21:35:00,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/e8f1a0f4882e4ffba03bb678822007df as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/e8f1a0f4882e4ffba03bb678822007df 2024-11-27T21:35:00,360 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/e8f1a0f4882e4ffba03bb678822007df, entries=100, sequenceid=16, filesize=9.4 K 2024-11-27T21:35:00,362 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 4c65e64f21dac3e5db22a4c47b796905 in 242ms, sequenceid=16, compaction requested=false 2024-11-27T21:35:00,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c65e64f21dac3e5db22a4c47b796905: 2024-11-27T21:35:00,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-27T21:35:00,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:00,398 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 4c65e64f21dac3e5db22a4c47b796905 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T21:35:00,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-27T21:35:00,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411278d9b6909826141c5980d4396f8282689_4c65e64f21dac3e5db22a4c47b796905 is 50, key is test_row_0/A:col10/1732743300172/Put/seqid=0 2024-11-27T21:35:00,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741972_1148 (size=12154) 2024-11-27T21:35:00,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:35:00,462 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411278d9b6909826141c5980d4396f8282689_4c65e64f21dac3e5db22a4c47b796905 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411278d9b6909826141c5980d4396f8282689_4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:00,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/08bddcfae5924aee9ded0e44a7e5dc84, store: [table=TestAcidGuarantees family=A region=4c65e64f21dac3e5db22a4c47b796905] 2024-11-27T21:35:00,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/08bddcfae5924aee9ded0e44a7e5dc84 is 175, key is test_row_0/A:col10/1732743300172/Put/seqid=0 2024-11-27T21:35:00,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741973_1149 (size=30955) 2024-11-27T21:35:00,483 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/08bddcfae5924aee9ded0e44a7e5dc84 2024-11-27T21:35:00,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/4521380bcc3e4f99ae4122505561aaa8 is 50, key is test_row_0/B:col10/1732743300172/Put/seqid=0 2024-11-27T21:35:00,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on 4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:00,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. as already flushing 2024-11-27T21:35:00,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741974_1150 (size=12001) 2024-11-27T21:35:00,519 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/4521380bcc3e4f99ae4122505561aaa8 2024-11-27T21:35:00,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/9ccb0ef2f24a46028ac556466bfe691e is 50, key is test_row_0/C:col10/1732743300172/Put/seqid=0 2024-11-27T21:35:00,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741975_1151 (size=12001) 2024-11-27T21:35:00,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:00,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743360512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:00,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:00,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743360512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:00,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:00,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:00,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:00,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743360559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:00,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:00,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743360559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:00,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:00,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:00,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743360559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:00,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:00,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:00,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:00,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743360664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:00,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:00,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743360664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:00,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:00,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:00,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:00,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743360674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:00,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:00,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743360674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:00,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:00,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743360674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:00,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:00,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:00,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:00,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-27T21:35:00,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:00,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743360873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:00,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:00,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743360874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:00,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:00,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:00,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:00,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743360894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:00,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:00,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743360894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:00,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:00,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:00,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743360894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:00,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:00,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:00,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:00,942 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/9ccb0ef2f24a46028ac556466bfe691e 2024-11-27T21:35:00,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/08bddcfae5924aee9ded0e44a7e5dc84 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/08bddcfae5924aee9ded0e44a7e5dc84 2024-11-27T21:35:00,956 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/08bddcfae5924aee9ded0e44a7e5dc84, entries=150, sequenceid=41, filesize=30.2 K 2024-11-27T21:35:00,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/4521380bcc3e4f99ae4122505561aaa8 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/4521380bcc3e4f99ae4122505561aaa8 2024-11-27T21:35:00,965 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/4521380bcc3e4f99ae4122505561aaa8, entries=150, sequenceid=41, filesize=11.7 K 2024-11-27T21:35:00,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/9ccb0ef2f24a46028ac556466bfe691e as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/9ccb0ef2f24a46028ac556466bfe691e 2024-11-27T21:35:00,971 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/9ccb0ef2f24a46028ac556466bfe691e, entries=150, sequenceid=41, filesize=11.7 K 2024-11-27T21:35:00,974 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 4c65e64f21dac3e5db22a4c47b796905 in 576ms, sequenceid=41, compaction requested=false 2024-11-27T21:35:00,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 4c65e64f21dac3e5db22a4c47b796905: 2024-11-27T21:35:00,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:00,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-27T21:35:00,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-27T21:35:00,981 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-27T21:35:00,981 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 891 msec 2024-11-27T21:35:00,986 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 902 msec 2024-11-27T21:35:01,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on 4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:01,187 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:35:01,187 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c65e64f21dac3e5db22a4c47b796905 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-27T21:35:01,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127a72ecb9cc0e44c8c8c8b3b0c11ca7048_4c65e64f21dac3e5db22a4c47b796905 is 50, key is test_row_0/A:col10/1732743301187/Put/seqid=0 2024-11-27T21:35:01,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-27T21:35:01,224 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:35:01,225 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:35:01,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-27T21:35:01,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-27T21:35:01,227 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:35:01,228 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:35:01,228 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:35:01,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:01,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743361236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:01,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:01,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:01,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743361237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:01,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:01,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:01,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743361240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:01,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:01,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:01,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743361242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:01,252 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,252 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,252 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:01,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:01,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743361245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:01,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:01,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741976_1152 (size=14594) 2024-11-27T21:35:01,262 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:35:01,269 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127a72ecb9cc0e44c8c8c8b3b0c11ca7048_4c65e64f21dac3e5db22a4c47b796905 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a72ecb9cc0e44c8c8c8b3b0c11ca7048_4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:01,270 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/4f7b4f1da64f46c684cd5a3bdc1cfc32, store: [table=TestAcidGuarantees family=A region=4c65e64f21dac3e5db22a4c47b796905] 2024-11-27T21:35:01,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/4f7b4f1da64f46c684cd5a3bdc1cfc32 is 175, key is test_row_0/A:col10/1732743301187/Put/seqid=0 2024-11-27T21:35:01,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741977_1153 (size=39549) 2024-11-27T21:35:01,284 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/4f7b4f1da64f46c684cd5a3bdc1cfc32 2024-11-27T21:35:01,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/9f3b068438e94d1ab4baab4c7d3e523b is 50, key is test_row_0/B:col10/1732743301187/Put/seqid=0 2024-11-27T21:35:01,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741978_1154 (size=12001) 2024-11-27T21:35:01,319 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/9f3b068438e94d1ab4baab4c7d3e523b 2024-11-27T21:35:01,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/227ab7724ffe40eb8a90cb7a808ba263 is 50, key is test_row_0/C:col10/1732743301187/Put/seqid=0 2024-11-27T21:35:01,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-27T21:35:01,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741979_1155 (size=12001) 2024-11-27T21:35:01,345 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/227ab7724ffe40eb8a90cb7a808ba263 2024-11-27T21:35:01,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:01,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743361353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:01,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:01,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743361353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:01,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:01,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743361353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:01,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:01,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:01,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:01,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/4f7b4f1da64f46c684cd5a3bdc1cfc32 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/4f7b4f1da64f46c684cd5a3bdc1cfc32 2024-11-27T21:35:01,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/4f7b4f1da64f46c684cd5a3bdc1cfc32, entries=200, sequenceid=54, filesize=38.6 K 2024-11-27T21:35:01,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/9f3b068438e94d1ab4baab4c7d3e523b as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/9f3b068438e94d1ab4baab4c7d3e523b 2024-11-27T21:35:01,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:01,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743361364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:01,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:01,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743361364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:01,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:01,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:01,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/9f3b068438e94d1ab4baab4c7d3e523b, entries=150, sequenceid=54, filesize=11.7 K 2024-11-27T21:35:01,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/227ab7724ffe40eb8a90cb7a808ba263 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/227ab7724ffe40eb8a90cb7a808ba263 2024-11-27T21:35:01,376 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/227ab7724ffe40eb8a90cb7a808ba263, entries=150, sequenceid=54, filesize=11.7 K 2024-11-27T21:35:01,377 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 4c65e64f21dac3e5db22a4c47b796905 in 190ms, sequenceid=54, compaction requested=true 2024-11-27T21:35:01,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c65e64f21dac3e5db22a4c47b796905: 2024-11-27T21:35:01,378 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:35:01,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c65e64f21dac3e5db22a4c47b796905:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:35:01,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:01,378 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:35:01,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c65e64f21dac3e5db22a4c47b796905:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:35:01,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:01,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c65e64f21dac3e5db22a4c47b796905:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:35:01,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:35:01,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-27T21:35:01,382 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127233 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:35:01,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:01,382 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): 4c65e64f21dac3e5db22a4c47b796905/A is initiating minor compaction (all files) 2024-11-27T21:35:01,382 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c65e64f21dac3e5db22a4c47b796905/A in TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:01,382 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 4c65e64f21dac3e5db22a4c47b796905 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-27T21:35:01,382 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/4ecf5c5ffdbe49d59968dcd0667d1087, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/08bddcfae5924aee9ded0e44a7e5dc84, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/4f7b4f1da64f46c684cd5a3bdc1cfc32] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp, totalSize=124.3 K 2024-11-27T21:35:01,382 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:01,383 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. files: [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/4ecf5c5ffdbe49d59968dcd0667d1087, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/08bddcfae5924aee9ded0e44a7e5dc84, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/4f7b4f1da64f46c684cd5a3bdc1cfc32] 2024-11-27T21:35:01,383 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ecf5c5ffdbe49d59968dcd0667d1087, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732743300105 2024-11-27T21:35:01,384 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08bddcfae5924aee9ded0e44a7e5dc84, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732743300169 2024-11-27T21:35:01,384 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f7b4f1da64f46c684cd5a3bdc1cfc32, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732743300508 2024-11-27T21:35:01,388 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:35:01,388 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): 4c65e64f21dac3e5db22a4c47b796905/B is initiating minor compaction (all files) 2024-11-27T21:35:01,388 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c65e64f21dac3e5db22a4c47b796905/B in TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:01,388 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/6d0e9bedae814a02a672fec49697df7c, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/4521380bcc3e4f99ae4122505561aaa8, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/9f3b068438e94d1ab4baab4c7d3e523b] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp, totalSize=32.9 K 2024-11-27T21:35:01,390 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d0e9bedae814a02a672fec49697df7c, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732743300114 2024-11-27T21:35:01,391 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 4521380bcc3e4f99ae4122505561aaa8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732743300169 2024-11-27T21:35:01,391 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f3b068438e94d1ab4baab4c7d3e523b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732743300533 2024-11-27T21:35:01,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127a97a09d1d2f24ed3bc416aeb7e2017e7_4c65e64f21dac3e5db22a4c47b796905 is 50, key is test_row_0/A:col10/1732743301215/Put/seqid=0 2024-11-27T21:35:01,409 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4c65e64f21dac3e5db22a4c47b796905] 2024-11-27T21:35:01,414 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c65e64f21dac3e5db22a4c47b796905#B#compaction#139 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:01,415 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/8393b83ac73248ce93b7ce82013bccf2 is 50, key is test_row_0/B:col10/1732743301187/Put/seqid=0 2024-11-27T21:35:01,418 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112766c6da5ab48e44faab3c2b1ed45f4957_4c65e64f21dac3e5db22a4c47b796905 store=[table=TestAcidGuarantees family=A region=4c65e64f21dac3e5db22a4c47b796905] 2024-11-27T21:35:01,425 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112766c6da5ab48e44faab3c2b1ed45f4957_4c65e64f21dac3e5db22a4c47b796905, store=[table=TestAcidGuarantees family=A region=4c65e64f21dac3e5db22a4c47b796905] 2024-11-27T21:35:01,425 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112766c6da5ab48e44faab3c2b1ed45f4957_4c65e64f21dac3e5db22a4c47b796905 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4c65e64f21dac3e5db22a4c47b796905] 2024-11-27T21:35:01,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741980_1156 (size=12154) 2024-11-27T21:35:01,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:35:01,447 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127a97a09d1d2f24ed3bc416aeb7e2017e7_4c65e64f21dac3e5db22a4c47b796905 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a97a09d1d2f24ed3bc416aeb7e2017e7_4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:01,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/d243c1e7b09f4ffc8794fa2995a29b4a, store: [table=TestAcidGuarantees family=A region=4c65e64f21dac3e5db22a4c47b796905] 2024-11-27T21:35:01,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/d243c1e7b09f4ffc8794fa2995a29b4a is 175, key is test_row_0/A:col10/1732743301215/Put/seqid=0 2024-11-27T21:35:01,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741981_1157 (size=12104) 2024-11-27T21:35:01,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741982_1158 (size=4469) 2024-11-27T21:35:01,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741983_1159 (size=30955) 2024-11-27T21:35:01,499 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/d243c1e7b09f4ffc8794fa2995a29b4a 2024-11-27T21:35:01,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/e3996131b4dd4340b44b5655754a152d is 50, key is test_row_0/B:col10/1732743301215/Put/seqid=0 2024-11-27T21:35:01,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741984_1160 (size=12001) 2024-11-27T21:35:01,528 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/e3996131b4dd4340b44b5655754a152d 2024-11-27T21:35:01,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/3a23bf4d7930473198a630fe6f9db4e2 is 50, key is test_row_0/C:col10/1732743301215/Put/seqid=0 2024-11-27T21:35:01,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-27T21:35:01,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741985_1161 (size=12001) 2024-11-27T21:35:01,557 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/3a23bf4d7930473198a630fe6f9db4e2 2024-11-27T21:35:01,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on 4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:01,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. as already flushing 2024-11-27T21:35:01,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/d243c1e7b09f4ffc8794fa2995a29b4a as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/d243c1e7b09f4ffc8794fa2995a29b4a 2024-11-27T21:35:01,575 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/d243c1e7b09f4ffc8794fa2995a29b4a, entries=150, sequenceid=77, filesize=30.2 K 2024-11-27T21:35:01,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/e3996131b4dd4340b44b5655754a152d as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/e3996131b4dd4340b44b5655754a152d 2024-11-27T21:35:01,586 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/e3996131b4dd4340b44b5655754a152d, entries=150, sequenceid=77, filesize=11.7 K 2024-11-27T21:35:01,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/3a23bf4d7930473198a630fe6f9db4e2 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/3a23bf4d7930473198a630fe6f9db4e2 2024-11-27T21:35:01,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:01,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743361579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:01,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:01,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:01,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743361580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:01,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:01,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:01,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743361582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:01,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:01,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:01,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743361584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:01,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:01,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:01,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743361588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:01,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:01,595 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/3a23bf4d7930473198a630fe6f9db4e2, entries=150, sequenceid=77, filesize=11.7 K 2024-11-27T21:35:01,597 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 4c65e64f21dac3e5db22a4c47b796905 in 215ms, sequenceid=77, compaction requested=true 2024-11-27T21:35:01,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 4c65e64f21dac3e5db22a4c47b796905: 2024-11-27T21:35:01,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:01,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-27T21:35:01,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-27T21:35:01,603 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-27T21:35:01,603 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 371 msec 2024-11-27T21:35:01,605 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 379 msec 2024-11-27T21:35:01,698 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:35:01,698 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c65e64f21dac3e5db22a4c47b796905 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-27T21:35:01,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on 4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:01,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112719b948bf18e74f45b462b21209368007_4c65e64f21dac3e5db22a4c47b796905 is 50, key is test_row_0/A:col10/1732743301697/Put/seqid=0 2024-11-27T21:35:01,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:01,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743361730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:01,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:01,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:01,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743361733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:01,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:01,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:01,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743361734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:01,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:01,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743361737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:01,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:01,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:01,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:01,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743361738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:01,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:01,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741986_1162 (size=17034) 2024-11-27T21:35:01,758 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T21:35:01,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:01,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:01,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743361844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:01,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743361844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:01,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:01,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:01,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-27T21:35:01,854 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:35:01,855 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:35:01,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:01,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743361854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:01,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:01,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:01,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743361854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:01,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-27T21:35:01,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:01,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:01,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743361855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:01,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:01,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:01,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-27T21:35:01,858 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:35:01,860 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:35:01,860 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:35:01,867 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/8393b83ac73248ce93b7ce82013bccf2 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/8393b83ac73248ce93b7ce82013bccf2 2024-11-27T21:35:01,876 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c65e64f21dac3e5db22a4c47b796905/B of 4c65e64f21dac3e5db22a4c47b796905 into 8393b83ac73248ce93b7ce82013bccf2(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:01,876 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c65e64f21dac3e5db22a4c47b796905: 2024-11-27T21:35:01,876 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., storeName=4c65e64f21dac3e5db22a4c47b796905/B, priority=13, startTime=1732743301378; duration=0sec 2024-11-27T21:35:01,876 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:35:01,876 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c65e64f21dac3e5db22a4c47b796905:B 2024-11-27T21:35:01,876 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:35:01,878 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:35:01,878 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): 4c65e64f21dac3e5db22a4c47b796905/C is initiating minor compaction (all files) 2024-11-27T21:35:01,878 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c65e64f21dac3e5db22a4c47b796905/C in TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:01,878 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/e8f1a0f4882e4ffba03bb678822007df, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/9ccb0ef2f24a46028ac556466bfe691e, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/227ab7724ffe40eb8a90cb7a808ba263, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/3a23bf4d7930473198a630fe6f9db4e2] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp, totalSize=44.6 K 2024-11-27T21:35:01,878 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting e8f1a0f4882e4ffba03bb678822007df, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732743300114 2024-11-27T21:35:01,879 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ccb0ef2f24a46028ac556466bfe691e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732743300169 2024-11-27T21:35:01,879 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 227ab7724ffe40eb8a90cb7a808ba263, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732743300533 2024-11-27T21:35:01,880 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a23bf4d7930473198a630fe6f9db4e2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732743301215 2024-11-27T21:35:01,882 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c65e64f21dac3e5db22a4c47b796905#A#compaction#138 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:01,882 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/7f84c81596a34a5a9e27abde5e185b87 is 175, key is test_row_0/A:col10/1732743301187/Put/seqid=0 2024-11-27T21:35:01,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741987_1163 (size=31058) 2024-11-27T21:35:01,898 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c65e64f21dac3e5db22a4c47b796905#C#compaction#143 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:01,898 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/03ad5007dd714131be9735a4f0c8bb7d is 50, key is test_row_0/C:col10/1732743301215/Put/seqid=0 2024-11-27T21:35:01,900 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/7f84c81596a34a5a9e27abde5e185b87 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/7f84c81596a34a5a9e27abde5e185b87 2024-11-27T21:35:01,907 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c65e64f21dac3e5db22a4c47b796905/A of 4c65e64f21dac3e5db22a4c47b796905 into 7f84c81596a34a5a9e27abde5e185b87(size=30.3 K), total size for store is 60.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:01,907 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c65e64f21dac3e5db22a4c47b796905: 2024-11-27T21:35:01,907 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., storeName=4c65e64f21dac3e5db22a4c47b796905/A, priority=13, startTime=1732743301378; duration=0sec 2024-11-27T21:35:01,907 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:01,907 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c65e64f21dac3e5db22a4c47b796905:A 2024-11-27T21:35:01,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741988_1164 (size=12139) 2024-11-27T21:35:01,917 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/03ad5007dd714131be9735a4f0c8bb7d as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/03ad5007dd714131be9735a4f0c8bb7d 2024-11-27T21:35:01,925 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4c65e64f21dac3e5db22a4c47b796905/C of 4c65e64f21dac3e5db22a4c47b796905 into 03ad5007dd714131be9735a4f0c8bb7d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:01,926 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c65e64f21dac3e5db22a4c47b796905: 2024-11-27T21:35:01,926 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., storeName=4c65e64f21dac3e5db22a4c47b796905/C, priority=12, startTime=1732743301382; duration=0sec 2024-11-27T21:35:01,926 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:01,926 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c65e64f21dac3e5db22a4c47b796905:C 2024-11-27T21:35:01,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-27T21:35:02,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-27T21:35:02,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:02,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. as already flushing 2024-11-27T21:35:02,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:02,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:02,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:02,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:02,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:02,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743362053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:02,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:02,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:02,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:02,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:02,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743362053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:02,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:02,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:02,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:02,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:02,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743362064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:02,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:02,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743362064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:02,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:02,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:02,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:02,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:02,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:02,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:02,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:02,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743362064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:02,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:02,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:02,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:02,158 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:35:02,163 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112719b948bf18e74f45b462b21209368007_4c65e64f21dac3e5db22a4c47b796905 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112719b948bf18e74f45b462b21209368007_4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:02,164 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/75b09b937f5644378763bde581cff786, store: [table=TestAcidGuarantees family=A region=4c65e64f21dac3e5db22a4c47b796905] 2024-11-27T21:35:02,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/75b09b937f5644378763bde581cff786 is 175, key is test_row_0/A:col10/1732743301697/Put/seqid=0 2024-11-27T21:35:02,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-27T21:35:02,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:02,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. as already flushing 2024-11-27T21:35:02,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:02,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:02,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:02,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:02,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-27T21:35:02,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741989_1165 (size=48139) 2024-11-27T21:35:02,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-27T21:35:02,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:02,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. as already flushing 2024-11-27T21:35:02,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:02,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:02,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:02,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:02,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:02,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743362363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:02,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:02,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:02,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743362363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:02,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:02,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:02,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:02,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:02,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:02,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:02,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743362373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:02,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:02,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743362373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:02,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:02,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:02,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:02,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:02,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:02,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743362373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:02,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:02,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:02,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:02,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:02,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:02,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-27T21:35:02,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:02,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. as already flushing 2024-11-27T21:35:02,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:02,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:02,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:02,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:02,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-27T21:35:02,596 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=92, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/75b09b937f5644378763bde581cff786 2024-11-27T21:35:02,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/cc501cf87b4242939a3b8f5aec8b9e20 is 50, key is test_row_0/B:col10/1732743301697/Put/seqid=0 2024-11-27T21:35:02,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741990_1166 (size=12001) 2024-11-27T21:35:02,617 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/cc501cf87b4242939a3b8f5aec8b9e20 2024-11-27T21:35:02,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/38750da6aa85410c83275ebdc5a6786b is 50, key is test_row_0/C:col10/1732743301697/Put/seqid=0 2024-11-27T21:35:02,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-27T21:35:02,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:02,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. as already flushing 2024-11-27T21:35:02,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:02,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:02,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:02,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:02,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741991_1167 (size=12001) 2024-11-27T21:35:02,638 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/38750da6aa85410c83275ebdc5a6786b 2024-11-27T21:35:02,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/75b09b937f5644378763bde581cff786 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/75b09b937f5644378763bde581cff786 2024-11-27T21:35:02,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/75b09b937f5644378763bde581cff786, entries=250, sequenceid=92, filesize=47.0 K 2024-11-27T21:35:02,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/cc501cf87b4242939a3b8f5aec8b9e20 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/cc501cf87b4242939a3b8f5aec8b9e20 2024-11-27T21:35:02,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/cc501cf87b4242939a3b8f5aec8b9e20, entries=150, sequenceid=92, filesize=11.7 K 2024-11-27T21:35:02,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/38750da6aa85410c83275ebdc5a6786b as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/38750da6aa85410c83275ebdc5a6786b 2024-11-27T21:35:02,679 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/38750da6aa85410c83275ebdc5a6786b, entries=150, sequenceid=92, filesize=11.7 K 2024-11-27T21:35:02,680 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 4c65e64f21dac3e5db22a4c47b796905 in 982ms, sequenceid=92, compaction requested=true 2024-11-27T21:35:02,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c65e64f21dac3e5db22a4c47b796905: 2024-11-27T21:35:02,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c65e64f21dac3e5db22a4c47b796905:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:35:02,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:02,680 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:35:02,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c65e64f21dac3e5db22a4c47b796905:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:35:02,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:35:02,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c65e64f21dac3e5db22a4c47b796905:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:35:02,680 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:35:02,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:35:02,682 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110152 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:35:02,682 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): 4c65e64f21dac3e5db22a4c47b796905/A is initiating minor compaction (all files) 2024-11-27T21:35:02,682 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c65e64f21dac3e5db22a4c47b796905/A in TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:02,682 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/7f84c81596a34a5a9e27abde5e185b87, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/d243c1e7b09f4ffc8794fa2995a29b4a, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/75b09b937f5644378763bde581cff786] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp, totalSize=107.6 K 2024-11-27T21:35:02,682 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:02,683 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. files: [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/7f84c81596a34a5a9e27abde5e185b87, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/d243c1e7b09f4ffc8794fa2995a29b4a, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/75b09b937f5644378763bde581cff786] 2024-11-27T21:35:02,684 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f84c81596a34a5a9e27abde5e185b87, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732743300533 2024-11-27T21:35:02,684 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:35:02,684 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): 4c65e64f21dac3e5db22a4c47b796905/B is initiating minor compaction (all files) 2024-11-27T21:35:02,684 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c65e64f21dac3e5db22a4c47b796905/B in TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:02,684 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/8393b83ac73248ce93b7ce82013bccf2, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/e3996131b4dd4340b44b5655754a152d, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/cc501cf87b4242939a3b8f5aec8b9e20] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp, totalSize=35.3 K 2024-11-27T21:35:02,685 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 8393b83ac73248ce93b7ce82013bccf2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732743300533 2024-11-27T21:35:02,685 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting d243c1e7b09f4ffc8794fa2995a29b4a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732743301215 2024-11-27T21:35:02,686 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting e3996131b4dd4340b44b5655754a152d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732743301215 2024-11-27T21:35:02,686 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75b09b937f5644378763bde581cff786, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732743301566 2024-11-27T21:35:02,686 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting cc501cf87b4242939a3b8f5aec8b9e20, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732743301577 2024-11-27T21:35:02,702 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c65e64f21dac3e5db22a4c47b796905#B#compaction#146 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:02,702 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/16da5bcc63164282842de3fbabe4df31 is 50, key is test_row_0/B:col10/1732743301697/Put/seqid=0 2024-11-27T21:35:02,710 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4c65e64f21dac3e5db22a4c47b796905] 2024-11-27T21:35:02,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741992_1168 (size=12207) 2024-11-27T21:35:02,727 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127d8bc961f35844a6cbfc07114c3e01eaf_4c65e64f21dac3e5db22a4c47b796905 store=[table=TestAcidGuarantees family=A region=4c65e64f21dac3e5db22a4c47b796905] 2024-11-27T21:35:02,732 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/16da5bcc63164282842de3fbabe4df31 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/16da5bcc63164282842de3fbabe4df31 2024-11-27T21:35:02,733 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127d8bc961f35844a6cbfc07114c3e01eaf_4c65e64f21dac3e5db22a4c47b796905, store=[table=TestAcidGuarantees family=A region=4c65e64f21dac3e5db22a4c47b796905] 2024-11-27T21:35:02,733 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d8bc961f35844a6cbfc07114c3e01eaf_4c65e64f21dac3e5db22a4c47b796905 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4c65e64f21dac3e5db22a4c47b796905] 2024-11-27T21:35:02,741 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c65e64f21dac3e5db22a4c47b796905/B of 4c65e64f21dac3e5db22a4c47b796905 into 16da5bcc63164282842de3fbabe4df31(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:02,741 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c65e64f21dac3e5db22a4c47b796905: 2024-11-27T21:35:02,741 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., storeName=4c65e64f21dac3e5db22a4c47b796905/B, priority=13, startTime=1732743302680; duration=0sec 2024-11-27T21:35:02,741 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:35:02,741 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c65e64f21dac3e5db22a4c47b796905:B 2024-11-27T21:35:02,741 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-27T21:35:02,744 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-27T21:35:02,744 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-27T21:35:02,744 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. because compaction request was cancelled 2024-11-27T21:35:02,744 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c65e64f21dac3e5db22a4c47b796905:C 2024-11-27T21:35:02,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741993_1169 (size=4469) 2024-11-27T21:35:02,761 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c65e64f21dac3e5db22a4c47b796905#A#compaction#147 average throughput is 0.48 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:02,762 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/903406b63d414a5da581436e2f6556af is 175, key is test_row_0/A:col10/1732743301697/Put/seqid=0 2024-11-27T21:35:02,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741994_1170 (size=31161) 2024-11-27T21:35:02,781 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/903406b63d414a5da581436e2f6556af as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/903406b63d414a5da581436e2f6556af 2024-11-27T21:35:02,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-27T21:35:02,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:02,784 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 4c65e64f21dac3e5db22a4c47b796905 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-27T21:35:02,790 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c65e64f21dac3e5db22a4c47b796905/A of 4c65e64f21dac3e5db22a4c47b796905 into 903406b63d414a5da581436e2f6556af(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:02,790 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c65e64f21dac3e5db22a4c47b796905: 2024-11-27T21:35:02,790 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., storeName=4c65e64f21dac3e5db22a4c47b796905/A, priority=13, startTime=1732743302680; duration=0sec 2024-11-27T21:35:02,790 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:02,792 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c65e64f21dac3e5db22a4c47b796905:A 2024-11-27T21:35:02,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127aa9892d6c76a48efa4aab25706f5da3d_4c65e64f21dac3e5db22a4c47b796905 is 50, key is test_row_0/A:col10/1732743301731/Put/seqid=0 2024-11-27T21:35:02,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741995_1171 (size=12154) 2024-11-27T21:35:02,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:35:02,817 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127aa9892d6c76a48efa4aab25706f5da3d_4c65e64f21dac3e5db22a4c47b796905 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127aa9892d6c76a48efa4aab25706f5da3d_4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:02,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/3064ddc776d4428a9a1d92895ba302be, store: [table=TestAcidGuarantees family=A region=4c65e64f21dac3e5db22a4c47b796905] 2024-11-27T21:35:02,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/3064ddc776d4428a9a1d92895ba302be is 175, key is test_row_0/A:col10/1732743301731/Put/seqid=0 2024-11-27T21:35:02,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741996_1172 (size=30955) 2024-11-27T21:35:02,846 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/3064ddc776d4428a9a1d92895ba302be 2024-11-27T21:35:02,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/67b5f27989b84b5e91d71e70a291cd24 is 50, key is test_row_0/B:col10/1732743301731/Put/seqid=0 2024-11-27T21:35:02,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741997_1173 (size=12001) 2024-11-27T21:35:02,870 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/67b5f27989b84b5e91d71e70a291cd24 2024-11-27T21:35:02,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on 4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:02,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. as already flushing 2024-11-27T21:35:02,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/16cbba6d7e6b4a4dbce2ac5ff4aa371d is 50, key is test_row_0/C:col10/1732743301731/Put/seqid=0 2024-11-27T21:35:02,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741998_1174 (size=12001) 2024-11-27T21:35:02,896 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/16cbba6d7e6b4a4dbce2ac5ff4aa371d 2024-11-27T21:35:02,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:02,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743362889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:02,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:02,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743362892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:02,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:02,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:02,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:02,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:02,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743362894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:02,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:02,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:02,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:02,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743362895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:02,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:02,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:02,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:02,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743362897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:02,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:02,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:02,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:02,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:02,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:02,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:02,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:02,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:02,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/3064ddc776d4428a9a1d92895ba302be as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/3064ddc776d4428a9a1d92895ba302be 2024-11-27T21:35:02,912 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/3064ddc776d4428a9a1d92895ba302be, entries=150, sequenceid=117, filesize=30.2 K 2024-11-27T21:35:02,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/67b5f27989b84b5e91d71e70a291cd24 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/67b5f27989b84b5e91d71e70a291cd24 2024-11-27T21:35:02,919 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/67b5f27989b84b5e91d71e70a291cd24, entries=150, sequenceid=117, filesize=11.7 K 2024-11-27T21:35:02,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/16cbba6d7e6b4a4dbce2ac5ff4aa371d as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/16cbba6d7e6b4a4dbce2ac5ff4aa371d 2024-11-27T21:35:02,931 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/16cbba6d7e6b4a4dbce2ac5ff4aa371d, entries=150, sequenceid=117, filesize=11.7 K 2024-11-27T21:35:02,933 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 4c65e64f21dac3e5db22a4c47b796905 in 148ms, sequenceid=117, compaction requested=true 2024-11-27T21:35:02,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 4c65e64f21dac3e5db22a4c47b796905: 2024-11-27T21:35:02,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:02,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-27T21:35:02,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-27T21:35:02,940 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-27T21:35:02,940 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0770 sec 2024-11-27T21:35:02,943 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.0860 sec 2024-11-27T21:35:02,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-27T21:35:02,993 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:35:02,994 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:35:02,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-27T21:35:02,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-27T21:35:02,998 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:35:02,999 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:35:02,999 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:35:03,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on 4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:03,025 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:35:03,025 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c65e64f21dac3e5db22a4c47b796905 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-27T21:35:03,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411279a05153aca5447728f9b6e0a6a49413a_4c65e64f21dac3e5db22a4c47b796905 is 50, key is test_row_0/A:col10/1732743303004/Put/seqid=0 2024-11-27T21:35:03,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741999_1175 (size=17284) 2024-11-27T21:35:03,043 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:35:03,049 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411279a05153aca5447728f9b6e0a6a49413a_4c65e64f21dac3e5db22a4c47b796905 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279a05153aca5447728f9b6e0a6a49413a_4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:03,052 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/3fbda9112e044405be549031773b5024, store: [table=TestAcidGuarantees family=A region=4c65e64f21dac3e5db22a4c47b796905] 2024-11-27T21:35:03,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/3fbda9112e044405be549031773b5024 is 175, key is test_row_0/A:col10/1732743303004/Put/seqid=0 2024-11-27T21:35:03,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:03,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743363052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:03,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:03,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:03,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:03,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:03,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743363054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:03,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:03,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743363058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:03,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:03,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:03,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:03,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:03,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743363058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:03,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:03,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:03,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:03,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:03,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:03,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:03,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:03,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743363065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:03,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:03,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:03,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:03,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742000_1176 (size=48389) 2024-11-27T21:35:03,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-27T21:35:03,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-27T21:35:03,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:03,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. as already flushing 2024-11-27T21:35:03,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:03,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:03,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:03,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:03,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:03,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743363163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:03,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:03,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:03,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:03,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:03,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743363163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:03,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:03,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:03,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:03,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743363163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:03,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:03,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:03,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:03,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:03,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:03,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743363163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:03,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:03,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:03,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:03,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:03,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743363173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:03,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:03,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:03,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:03,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-27T21:35:03,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:03,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. as already flushing 2024-11-27T21:35:03,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:03,310 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:03,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:03,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:03,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-27T21:35:03,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:03,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743363373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:03,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:03,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:03,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743363373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:03,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743363373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:03,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:03,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:03,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:03,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:03,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:03,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:03,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:03,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:03,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:03,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743363374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:03,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:03,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:03,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:03,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:03,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:03,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743363383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:03,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:03,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:03,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:03,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-27T21:35:03,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:03,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. as already flushing 2024-11-27T21:35:03,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:03,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:03,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:03,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:03,478 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=136, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/3fbda9112e044405be549031773b5024 2024-11-27T21:35:03,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/56ce1f6e5a50484cb54dfb3dbdcceea4 is 50, key is test_row_0/B:col10/1732743303004/Put/seqid=0 2024-11-27T21:35:03,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742001_1177 (size=12151) 2024-11-27T21:35:03,494 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/56ce1f6e5a50484cb54dfb3dbdcceea4 2024-11-27T21:35:03,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/67bfa5beaa84419db37707b9bb717d2f is 50, key is test_row_0/C:col10/1732743303004/Put/seqid=0 2024-11-27T21:35:03,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742002_1178 (size=12151) 2024-11-27T21:35:03,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-27T21:35:03,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:03,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. as already flushing 2024-11-27T21:35:03,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:03,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:03,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:03,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:03,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-27T21:35:03,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:03,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:03,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743363683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:03,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743363684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:03,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:03,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:03,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:03,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:03,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:03,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743363684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:03,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:03,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743363684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:03,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:03,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:03,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:03,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:03,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:03,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:03,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:03,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:03,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:03,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743363693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:03,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:03,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:03,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:03,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-27T21:35:03,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:03,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. as already flushing 2024-11-27T21:35:03,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:03,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:03,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:03,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:03,907 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/67bfa5beaa84419db37707b9bb717d2f 2024-11-27T21:35:03,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/3fbda9112e044405be549031773b5024 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/3fbda9112e044405be549031773b5024 2024-11-27T21:35:03,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/3fbda9112e044405be549031773b5024, entries=250, sequenceid=136, filesize=47.3 K 2024-11-27T21:35:03,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/56ce1f6e5a50484cb54dfb3dbdcceea4 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/56ce1f6e5a50484cb54dfb3dbdcceea4 2024-11-27T21:35:03,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-27T21:35:03,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:03,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. as already flushing 2024-11-27T21:35:03,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:03,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:03,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:03,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:03,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/56ce1f6e5a50484cb54dfb3dbdcceea4, entries=150, sequenceid=136, filesize=11.9 K 2024-11-27T21:35:03,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/67bfa5beaa84419db37707b9bb717d2f as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/67bfa5beaa84419db37707b9bb717d2f 2024-11-27T21:35:03,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/67bfa5beaa84419db37707b9bb717d2f, entries=150, sequenceid=136, filesize=11.9 K 2024-11-27T21:35:03,936 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for 4c65e64f21dac3e5db22a4c47b796905 in 911ms, sequenceid=136, compaction requested=true 2024-11-27T21:35:03,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c65e64f21dac3e5db22a4c47b796905: 2024-11-27T21:35:03,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c65e64f21dac3e5db22a4c47b796905:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:35:03,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:03,936 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:35:03,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c65e64f21dac3e5db22a4c47b796905:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:35:03,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:03,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c65e64f21dac3e5db22a4c47b796905:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:35:03,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:35:03,936 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:35:03,938 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:35:03,938 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110505 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:35:03,938 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): 4c65e64f21dac3e5db22a4c47b796905/B is initiating minor compaction (all files) 2024-11-27T21:35:03,938 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): 4c65e64f21dac3e5db22a4c47b796905/A is initiating minor compaction (all files) 2024-11-27T21:35:03,938 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c65e64f21dac3e5db22a4c47b796905/B in TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:03,938 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c65e64f21dac3e5db22a4c47b796905/A in TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:03,938 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/16da5bcc63164282842de3fbabe4df31, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/67b5f27989b84b5e91d71e70a291cd24, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/56ce1f6e5a50484cb54dfb3dbdcceea4] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp, totalSize=35.5 K 2024-11-27T21:35:03,938 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/903406b63d414a5da581436e2f6556af, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/3064ddc776d4428a9a1d92895ba302be, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/3fbda9112e044405be549031773b5024] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp, totalSize=107.9 K 2024-11-27T21:35:03,938 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:03,938 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. files: [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/903406b63d414a5da581436e2f6556af, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/3064ddc776d4428a9a1d92895ba302be, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/3fbda9112e044405be549031773b5024] 2024-11-27T21:35:03,939 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 16da5bcc63164282842de3fbabe4df31, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732743301577 2024-11-27T21:35:03,939 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 903406b63d414a5da581436e2f6556af, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732743301577 2024-11-27T21:35:03,940 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 67b5f27989b84b5e91d71e70a291cd24, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732743301727 2024-11-27T21:35:03,940 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3064ddc776d4428a9a1d92895ba302be, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732743301727 2024-11-27T21:35:03,940 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 56ce1f6e5a50484cb54dfb3dbdcceea4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732743302893 2024-11-27T21:35:03,940 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fbda9112e044405be549031773b5024, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732743302892 2024-11-27T21:35:03,958 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4c65e64f21dac3e5db22a4c47b796905] 2024-11-27T21:35:03,959 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c65e64f21dac3e5db22a4c47b796905#B#compaction#155 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:03,960 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/688e026109224ae6beefeea5a5a74107 is 50, key is test_row_0/B:col10/1732743303004/Put/seqid=0 2024-11-27T21:35:03,969 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112720eca8d23bf94be99ab3e7f4b94712c3_4c65e64f21dac3e5db22a4c47b796905 store=[table=TestAcidGuarantees family=A region=4c65e64f21dac3e5db22a4c47b796905] 2024-11-27T21:35:03,971 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112720eca8d23bf94be99ab3e7f4b94712c3_4c65e64f21dac3e5db22a4c47b796905, store=[table=TestAcidGuarantees family=A region=4c65e64f21dac3e5db22a4c47b796905] 2024-11-27T21:35:03,971 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112720eca8d23bf94be99ab3e7f4b94712c3_4c65e64f21dac3e5db22a4c47b796905 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4c65e64f21dac3e5db22a4c47b796905] 2024-11-27T21:35:03,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742003_1179 (size=12459) 2024-11-27T21:35:03,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742004_1180 (size=4469) 2024-11-27T21:35:04,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-27T21:35:04,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:04,079 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 4c65e64f21dac3e5db22a4c47b796905 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-27T21:35:04,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112734cd4d2aba1f438aa743b7e52fe8a0b9_4c65e64f21dac3e5db22a4c47b796905 is 50, key is test_row_0/A:col10/1732743303055/Put/seqid=0 2024-11-27T21:35:04,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742005_1181 (size=12304) 2024-11-27T21:35:04,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-27T21:35:04,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on 4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:04,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. as already flushing 2024-11-27T21:35:04,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:04,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743364211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:04,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:04,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743364212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:04,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:04,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:04,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:04,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:04,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:04,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:04,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:04,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743364214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:04,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:04,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:04,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:04,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:04,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743364215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:04,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:04,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:04,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743364216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:04,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:04,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:04,222 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:04,222 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:04,222 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:04,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:04,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743364323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:04,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:04,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:04,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:04,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:04,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743364323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:04,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:04,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:04,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:04,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743364324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:04,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:04,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:04,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:04,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:04,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:04,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743364333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:04,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:04,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:04,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:04,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:04,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743364333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:04,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:04,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:04,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:04,382 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/688e026109224ae6beefeea5a5a74107 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/688e026109224ae6beefeea5a5a74107 2024-11-27T21:35:04,387 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c65e64f21dac3e5db22a4c47b796905/B of 4c65e64f21dac3e5db22a4c47b796905 into 688e026109224ae6beefeea5a5a74107(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:04,387 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c65e64f21dac3e5db22a4c47b796905: 2024-11-27T21:35:04,387 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., storeName=4c65e64f21dac3e5db22a4c47b796905/B, priority=13, startTime=1732743303936; duration=0sec 2024-11-27T21:35:04,388 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:35:04,388 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c65e64f21dac3e5db22a4c47b796905:B 2024-11-27T21:35:04,388 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:35:04,389 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48292 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:35:04,389 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): 4c65e64f21dac3e5db22a4c47b796905/C is initiating minor compaction (all files) 2024-11-27T21:35:04,389 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c65e64f21dac3e5db22a4c47b796905/C in TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:04,389 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/03ad5007dd714131be9735a4f0c8bb7d, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/38750da6aa85410c83275ebdc5a6786b, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/16cbba6d7e6b4a4dbce2ac5ff4aa371d, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/67bfa5beaa84419db37707b9bb717d2f] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp, totalSize=47.2 K 2024-11-27T21:35:04,390 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 03ad5007dd714131be9735a4f0c8bb7d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732743301215 2024-11-27T21:35:04,390 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 38750da6aa85410c83275ebdc5a6786b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732743301577 2024-11-27T21:35:04,391 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 16cbba6d7e6b4a4dbce2ac5ff4aa371d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732743301727 2024-11-27T21:35:04,391 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 67bfa5beaa84419db37707b9bb717d2f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732743302893 2024-11-27T21:35:04,394 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c65e64f21dac3e5db22a4c47b796905#A#compaction#154 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:04,394 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/a3945c7d73714ab7996cc0b2608c7a99 is 175, key is test_row_0/A:col10/1732743303004/Put/seqid=0 2024-11-27T21:35:04,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742006_1182 (size=31413) 2024-11-27T21:35:04,407 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c65e64f21dac3e5db22a4c47b796905#C#compaction#157 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:04,407 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/3830173ad7874577b1265f4c4cc578ee is 50, key is test_row_0/C:col10/1732743303004/Put/seqid=0 2024-11-27T21:35:04,412 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/a3945c7d73714ab7996cc0b2608c7a99 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/a3945c7d73714ab7996cc0b2608c7a99 2024-11-27T21:35:04,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742007_1183 (size=12425) 2024-11-27T21:35:04,418 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c65e64f21dac3e5db22a4c47b796905/A of 4c65e64f21dac3e5db22a4c47b796905 into a3945c7d73714ab7996cc0b2608c7a99(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:04,418 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c65e64f21dac3e5db22a4c47b796905: 2024-11-27T21:35:04,419 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., storeName=4c65e64f21dac3e5db22a4c47b796905/A, priority=13, startTime=1732743303936; duration=0sec 2024-11-27T21:35:04,419 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:04,419 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c65e64f21dac3e5db22a4c47b796905:A 2024-11-27T21:35:04,423 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/3830173ad7874577b1265f4c4cc578ee as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/3830173ad7874577b1265f4c4cc578ee 2024-11-27T21:35:04,431 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4c65e64f21dac3e5db22a4c47b796905/C of 4c65e64f21dac3e5db22a4c47b796905 into 3830173ad7874577b1265f4c4cc578ee(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:04,431 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c65e64f21dac3e5db22a4c47b796905: 2024-11-27T21:35:04,431 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., storeName=4c65e64f21dac3e5db22a4c47b796905/C, priority=12, startTime=1732743303936; duration=0sec 2024-11-27T21:35:04,432 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:04,432 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c65e64f21dac3e5db22a4c47b796905:C 2024-11-27T21:35:04,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:35:04,512 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112734cd4d2aba1f438aa743b7e52fe8a0b9_4c65e64f21dac3e5db22a4c47b796905 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112734cd4d2aba1f438aa743b7e52fe8a0b9_4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:04,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/e2767feb957e4a699ed74830bb5d16cf, store: [table=TestAcidGuarantees family=A region=4c65e64f21dac3e5db22a4c47b796905] 2024-11-27T21:35:04,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/e2767feb957e4a699ed74830bb5d16cf is 175, key is test_row_0/A:col10/1732743303055/Put/seqid=0 2024-11-27T21:35:04,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742008_1184 (size=31105) 2024-11-27T21:35:04,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:04,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743364534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:04,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:04,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:04,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:04,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743364534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:04,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:04,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:04,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743364534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:04,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:04,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:04,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:04,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:04,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:04,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:04,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:04,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743364543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:04,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:04,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:04,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743364543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:04,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:04,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:04,546 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:04,546 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:04,546 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:04,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:04,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743364843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:04,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:04,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:04,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:04,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:04,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:04,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743364843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:04,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743364843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:04,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:04,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:04,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:04,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:04,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:04,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:04,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:04,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743364853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:04,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:04,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:04,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:04,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:04,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743364854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:04,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:04,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:04,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:04,928 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/e2767feb957e4a699ed74830bb5d16cf 2024-11-27T21:35:04,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/5ed2fdd940fa4ad0a47a8ce8d8f026a9 is 50, key is test_row_0/B:col10/1732743303055/Put/seqid=0 2024-11-27T21:35:04,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742009_1185 (size=12151) 2024-11-27T21:35:04,947 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/5ed2fdd940fa4ad0a47a8ce8d8f026a9 2024-11-27T21:35:04,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/ff492250e8274e4193d935c5f4451009 is 50, key is test_row_0/C:col10/1732743303055/Put/seqid=0 2024-11-27T21:35:04,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742010_1186 (size=12151) 2024-11-27T21:35:04,965 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/ff492250e8274e4193d935c5f4451009 2024-11-27T21:35:04,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/e2767feb957e4a699ed74830bb5d16cf as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/e2767feb957e4a699ed74830bb5d16cf 2024-11-27T21:35:04,975 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/e2767feb957e4a699ed74830bb5d16cf, entries=150, sequenceid=155, filesize=30.4 K 2024-11-27T21:35:04,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/5ed2fdd940fa4ad0a47a8ce8d8f026a9 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/5ed2fdd940fa4ad0a47a8ce8d8f026a9 2024-11-27T21:35:04,981 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/5ed2fdd940fa4ad0a47a8ce8d8f026a9, entries=150, sequenceid=155, filesize=11.9 K 2024-11-27T21:35:04,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/ff492250e8274e4193d935c5f4451009 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/ff492250e8274e4193d935c5f4451009 2024-11-27T21:35:04,988 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/ff492250e8274e4193d935c5f4451009, entries=150, sequenceid=155, filesize=11.9 K 2024-11-27T21:35:04,989 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for 4c65e64f21dac3e5db22a4c47b796905 in 910ms, sequenceid=155, compaction requested=false 2024-11-27T21:35:04,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 4c65e64f21dac3e5db22a4c47b796905: 2024-11-27T21:35:04,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:04,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-27T21:35:04,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-27T21:35:04,992 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-27T21:35:04,992 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9910 sec 2024-11-27T21:35:04,994 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.9990 sec 2024-11-27T21:35:05,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-27T21:35:05,144 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:35:05,144 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:35:05,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-27T21:35:05,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-27T21:35:05,147 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:35:05,147 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:35:05,148 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:35:05,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-27T21:35:05,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-27T21:35:05,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:05,301 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 4c65e64f21dac3e5db22a4c47b796905 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-27T21:35:05,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411276feb782ffaa7455697ba28dd86d964af_4c65e64f21dac3e5db22a4c47b796905 is 50, key is test_row_0/A:col10/1732743304208/Put/seqid=0 2024-11-27T21:35:05,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742011_1187 (size=12304) 2024-11-27T21:35:05,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:35:05,322 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411276feb782ffaa7455697ba28dd86d964af_4c65e64f21dac3e5db22a4c47b796905 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411276feb782ffaa7455697ba28dd86d964af_4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:05,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/ada7412ddd6144ca8ef648bfa13aa80e, store: [table=TestAcidGuarantees family=A region=4c65e64f21dac3e5db22a4c47b796905] 2024-11-27T21:35:05,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/ada7412ddd6144ca8ef648bfa13aa80e is 175, key is test_row_0/A:col10/1732743304208/Put/seqid=0 2024-11-27T21:35:05,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742012_1188 (size=31105) 2024-11-27T21:35:05,329 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=176, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/ada7412ddd6144ca8ef648bfa13aa80e 2024-11-27T21:35:05,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/bbfb5343fbe1473b97537d9f8dd81570 is 50, key is test_row_0/B:col10/1732743304208/Put/seqid=0 2024-11-27T21:35:05,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742013_1189 (size=12151) 2024-11-27T21:35:05,346 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/bbfb5343fbe1473b97537d9f8dd81570 2024-11-27T21:35:05,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on 4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:05,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. as already flushing 2024-11-27T21:35:05,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/13a67ca3871c434ab6eb034636ed9b3e is 50, key is test_row_0/C:col10/1732743304208/Put/seqid=0 2024-11-27T21:35:05,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742014_1190 (size=12151) 2024-11-27T21:35:05,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:05,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743365375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:05,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:05,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:05,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:05,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:05,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743365379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:05,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:05,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:05,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:05,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:05,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743365380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:05,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:05,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743365382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:05,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:05,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743365382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:05,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:05,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:05,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:05,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:05,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:05,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:05,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:05,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:05,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:05,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-27T21:35:05,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:05,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743365493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:05,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:05,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743365494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:05,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:05,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:05,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:05,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:05,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:05,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:05,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743365494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:05,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:05,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:05,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:05,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:05,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:05,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743365494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:05,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:05,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:05,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:05,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743365494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:05,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:05,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:05,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:05,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:05,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:05,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743365704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:05,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:05,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743365704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:05,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:05,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:05,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:05,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:05,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:05,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743365704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:05,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:05,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:05,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:05,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:05,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743365704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:05,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:05,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:05,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:05,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:05,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:05,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743365704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:05,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:05,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:05,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:05,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:05,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-27T21:35:05,776 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/13a67ca3871c434ab6eb034636ed9b3e 2024-11-27T21:35:05,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/ada7412ddd6144ca8ef648bfa13aa80e as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/ada7412ddd6144ca8ef648bfa13aa80e 2024-11-27T21:35:05,787 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/ada7412ddd6144ca8ef648bfa13aa80e, entries=150, sequenceid=176, filesize=30.4 K 2024-11-27T21:35:05,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/bbfb5343fbe1473b97537d9f8dd81570 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/bbfb5343fbe1473b97537d9f8dd81570 2024-11-27T21:35:05,792 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/bbfb5343fbe1473b97537d9f8dd81570, entries=150, sequenceid=176, filesize=11.9 K 2024-11-27T21:35:05,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/13a67ca3871c434ab6eb034636ed9b3e as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/13a67ca3871c434ab6eb034636ed9b3e 2024-11-27T21:35:05,798 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/13a67ca3871c434ab6eb034636ed9b3e, entries=150, sequenceid=176, filesize=11.9 K 2024-11-27T21:35:05,799 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 4c65e64f21dac3e5db22a4c47b796905 in 497ms, sequenceid=176, compaction requested=true 2024-11-27T21:35:05,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 4c65e64f21dac3e5db22a4c47b796905: 2024-11-27T21:35:05,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:05,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-27T21:35:05,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-27T21:35:05,802 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-27T21:35:05,803 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 652 msec 2024-11-27T21:35:05,805 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 659 msec 2024-11-27T21:35:06,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on 4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:06,016 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:35:06,016 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c65e64f21dac3e5db22a4c47b796905 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-27T21:35:06,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d5277ff6cfed47928d0120fcbdd39106_4c65e64f21dac3e5db22a4c47b796905 is 50, key is test_row_0/A:col10/1732743305380/Put/seqid=0 2024-11-27T21:35:06,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:06,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:06,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743366030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:06,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743366031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:06,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:06,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743366036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:06,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:06,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:06,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:06,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:06,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:06,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:06,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:06,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:06,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:06,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742015_1191 (size=17284) 2024-11-27T21:35:06,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:06,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743366038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:06,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:06,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:06,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:06,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743366038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:06,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:06,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:06,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:06,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:06,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:06,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743366143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:06,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:06,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:06,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:06,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743366144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:06,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:06,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:06,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:06,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743366144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:06,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:06,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:06,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:06,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:06,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:06,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:06,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743366153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:06,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:06,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:06,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743366153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:06,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:06,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:06,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:06,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:06,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-27T21:35:06,283 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:35:06,284 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:35:06,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-27T21:35:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-27T21:35:06,286 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:35:06,287 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:35:06,287 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:35:06,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743366353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:06,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:06,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743366353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:06,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:06,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:06,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743366353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:06,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:06,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:06,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:06,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:06,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:06,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:06,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:06,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743366363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:06,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:06,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:06,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:06,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:06,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743366363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:06,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:06,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:06,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:06,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-27T21:35:06,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-27T21:35:06,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:06,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. as already flushing 2024-11-27T21:35:06,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:06,440 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:06,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:06,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:06,445 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:35:06,450 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d5277ff6cfed47928d0120fcbdd39106_4c65e64f21dac3e5db22a4c47b796905 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d5277ff6cfed47928d0120fcbdd39106_4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:06,451 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/4a8787c5c2614022bd929509aa03dbe7, store: [table=TestAcidGuarantees family=A region=4c65e64f21dac3e5db22a4c47b796905] 2024-11-27T21:35:06,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/4a8787c5c2614022bd929509aa03dbe7 is 175, key is test_row_0/A:col10/1732743305380/Put/seqid=0 2024-11-27T21:35:06,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742016_1192 (size=48389) 2024-11-27T21:35:06,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-27T21:35:06,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:06,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. as already flushing 2024-11-27T21:35:06,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:06,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:06,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:06,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:06,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-27T21:35:06,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:06,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743366663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:06,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:06,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:06,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:06,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:06,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743366663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:06,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:06,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:06,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:06,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:06,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743366663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:06,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:06,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:06,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:06,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:06,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743366674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:06,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:06,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:06,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:06,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:06,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743366674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:06,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:06,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:06,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:06,698 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-27T21:35:06,698 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-27T21:35:06,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-27T21:35:06,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:06,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. as already flushing 2024-11-27T21:35:06,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:06,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:06,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:06,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:06,870 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=196, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/4a8787c5c2614022bd929509aa03dbe7 2024-11-27T21:35:06,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/18744fd273eb41f8b4b8061afba6bf44 is 50, key is test_row_0/B:col10/1732743305380/Put/seqid=0 2024-11-27T21:35:06,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742017_1193 (size=12151) 2024-11-27T21:35:06,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-27T21:35:06,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:06,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. as already flushing 2024-11-27T21:35:06,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:06,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:06,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:06,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:06,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-27T21:35:07,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-27T21:35:07,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:07,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. as already flushing 2024-11-27T21:35:07,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:07,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:07,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:07,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:07,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:07,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743367174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:07,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:07,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:07,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:07,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:07,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743367183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:07,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:07,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:07,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:07,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:07,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743367183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:07,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:07,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:07,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:07,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:07,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743367183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:07,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:07,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743367183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:07,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:07,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:07,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:07,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:07,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:07,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:07,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-27T21:35:07,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:07,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. as already flushing 2024-11-27T21:35:07,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:07,207 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:07,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:07,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:07,282 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/18744fd273eb41f8b4b8061afba6bf44 2024-11-27T21:35:07,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/ecb58b15d0714b26827d10f3397a1a56 is 50, key is test_row_0/C:col10/1732743305380/Put/seqid=0 2024-11-27T21:35:07,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742018_1194 (size=12151) 2024-11-27T21:35:07,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-27T21:35:07,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:07,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. as already flushing 2024-11-27T21:35:07,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:07,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:07,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:07,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:07,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-27T21:35:07,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-27T21:35:07,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:07,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. as already flushing 2024-11-27T21:35:07,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:07,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:07,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:07,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:07,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-27T21:35:07,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:07,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. as already flushing 2024-11-27T21:35:07,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:07,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:07,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:07,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:07,697 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/ecb58b15d0714b26827d10f3397a1a56 2024-11-27T21:35:07,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/4a8787c5c2614022bd929509aa03dbe7 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/4a8787c5c2614022bd929509aa03dbe7 2024-11-27T21:35:07,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/4a8787c5c2614022bd929509aa03dbe7, entries=250, sequenceid=196, filesize=47.3 K 2024-11-27T21:35:07,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/18744fd273eb41f8b4b8061afba6bf44 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/18744fd273eb41f8b4b8061afba6bf44 2024-11-27T21:35:07,712 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/18744fd273eb41f8b4b8061afba6bf44, entries=150, sequenceid=196, filesize=11.9 K 2024-11-27T21:35:07,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/ecb58b15d0714b26827d10f3397a1a56 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/ecb58b15d0714b26827d10f3397a1a56 2024-11-27T21:35:07,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/ecb58b15d0714b26827d10f3397a1a56, entries=150, sequenceid=196, filesize=11.9 K 2024-11-27T21:35:07,720 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 4c65e64f21dac3e5db22a4c47b796905 in 1703ms, sequenceid=196, compaction requested=true 2024-11-27T21:35:07,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c65e64f21dac3e5db22a4c47b796905: 2024-11-27T21:35:07,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c65e64f21dac3e5db22a4c47b796905:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:35:07,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:07,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c65e64f21dac3e5db22a4c47b796905:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:35:07,720 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:35:07,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:07,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c65e64f21dac3e5db22a4c47b796905:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:35:07,720 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:35:07,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:35:07,721 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142012 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:35:07,721 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:35:07,721 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): 4c65e64f21dac3e5db22a4c47b796905/A is initiating minor compaction (all files) 2024-11-27T21:35:07,721 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): 4c65e64f21dac3e5db22a4c47b796905/B is initiating minor compaction (all files) 2024-11-27T21:35:07,721 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c65e64f21dac3e5db22a4c47b796905/A in TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:07,721 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c65e64f21dac3e5db22a4c47b796905/B in TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:07,722 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/688e026109224ae6beefeea5a5a74107, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/5ed2fdd940fa4ad0a47a8ce8d8f026a9, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/bbfb5343fbe1473b97537d9f8dd81570, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/18744fd273eb41f8b4b8061afba6bf44] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp, totalSize=47.8 K 2024-11-27T21:35:07,722 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/a3945c7d73714ab7996cc0b2608c7a99, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/e2767feb957e4a699ed74830bb5d16cf, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/ada7412ddd6144ca8ef648bfa13aa80e, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/4a8787c5c2614022bd929509aa03dbe7] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp, totalSize=138.7 K 2024-11-27T21:35:07,722 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:07,722 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. files: [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/a3945c7d73714ab7996cc0b2608c7a99, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/e2767feb957e4a699ed74830bb5d16cf, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/ada7412ddd6144ca8ef648bfa13aa80e, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/4a8787c5c2614022bd929509aa03dbe7] 2024-11-27T21:35:07,722 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 688e026109224ae6beefeea5a5a74107, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732743302893 2024-11-27T21:35:07,722 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3945c7d73714ab7996cc0b2608c7a99, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732743302893 2024-11-27T21:35:07,722 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ed2fdd940fa4ad0a47a8ce8d8f026a9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732743303051 2024-11-27T21:35:07,723 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting bbfb5343fbe1473b97537d9f8dd81570, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732743304208 2024-11-27T21:35:07,723 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2767feb957e4a699ed74830bb5d16cf, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732743303051 2024-11-27T21:35:07,723 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 18744fd273eb41f8b4b8061afba6bf44, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732743305377 2024-11-27T21:35:07,723 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting ada7412ddd6144ca8ef648bfa13aa80e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732743304208 2024-11-27T21:35:07,724 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a8787c5c2614022bd929509aa03dbe7, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732743305364 2024-11-27T21:35:07,734 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4c65e64f21dac3e5db22a4c47b796905] 2024-11-27T21:35:07,734 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c65e64f21dac3e5db22a4c47b796905#B#compaction#166 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:07,735 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/eac5727c995542eea7b66a925ae703e3 is 50, key is test_row_0/B:col10/1732743305380/Put/seqid=0 2024-11-27T21:35:07,737 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411272003f643d929443a9c3dded90cb82ac0_4c65e64f21dac3e5db22a4c47b796905 store=[table=TestAcidGuarantees family=A region=4c65e64f21dac3e5db22a4c47b796905] 2024-11-27T21:35:07,740 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411272003f643d929443a9c3dded90cb82ac0_4c65e64f21dac3e5db22a4c47b796905, store=[table=TestAcidGuarantees family=A region=4c65e64f21dac3e5db22a4c47b796905] 2024-11-27T21:35:07,740 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411272003f643d929443a9c3dded90cb82ac0_4c65e64f21dac3e5db22a4c47b796905 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4c65e64f21dac3e5db22a4c47b796905] 2024-11-27T21:35:07,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742019_1195 (size=12595) 2024-11-27T21:35:07,759 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/eac5727c995542eea7b66a925ae703e3 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/eac5727c995542eea7b66a925ae703e3 2024-11-27T21:35:07,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742020_1196 (size=4469) 2024-11-27T21:35:07,766 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4c65e64f21dac3e5db22a4c47b796905/B of 4c65e64f21dac3e5db22a4c47b796905 into eac5727c995542eea7b66a925ae703e3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:07,766 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c65e64f21dac3e5db22a4c47b796905: 2024-11-27T21:35:07,766 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., storeName=4c65e64f21dac3e5db22a4c47b796905/B, priority=12, startTime=1732743307720; duration=0sec 2024-11-27T21:35:07,766 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:35:07,766 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c65e64f21dac3e5db22a4c47b796905:B 2024-11-27T21:35:07,766 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:35:07,768 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48878 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:35:07,768 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): 4c65e64f21dac3e5db22a4c47b796905/C is initiating minor compaction (all files) 2024-11-27T21:35:07,768 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c65e64f21dac3e5db22a4c47b796905/C in TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:07,768 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/3830173ad7874577b1265f4c4cc578ee, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/ff492250e8274e4193d935c5f4451009, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/13a67ca3871c434ab6eb034636ed9b3e, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/ecb58b15d0714b26827d10f3397a1a56] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp, totalSize=47.7 K 2024-11-27T21:35:07,769 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 3830173ad7874577b1265f4c4cc578ee, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732743302893 2024-11-27T21:35:07,769 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting ff492250e8274e4193d935c5f4451009, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732743303051 2024-11-27T21:35:07,770 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 13a67ca3871c434ab6eb034636ed9b3e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732743304208 2024-11-27T21:35:07,770 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting ecb58b15d0714b26827d10f3397a1a56, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732743305377 2024-11-27T21:35:07,779 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c65e64f21dac3e5db22a4c47b796905#C#compaction#168 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:07,780 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/849bb24a746146e1883b3e335fbd7dc6 is 50, key is test_row_0/C:col10/1732743305380/Put/seqid=0 2024-11-27T21:35:07,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742021_1197 (size=12561) 2024-11-27T21:35:07,789 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/849bb24a746146e1883b3e335fbd7dc6 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/849bb24a746146e1883b3e335fbd7dc6 2024-11-27T21:35:07,795 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4c65e64f21dac3e5db22a4c47b796905/C of 4c65e64f21dac3e5db22a4c47b796905 into 849bb24a746146e1883b3e335fbd7dc6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:07,795 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c65e64f21dac3e5db22a4c47b796905: 2024-11-27T21:35:07,795 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., storeName=4c65e64f21dac3e5db22a4c47b796905/C, priority=12, startTime=1732743307720; duration=0sec 2024-11-27T21:35:07,795 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:07,795 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c65e64f21dac3e5db22a4c47b796905:C 2024-11-27T21:35:07,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-27T21:35:07,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:07,820 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 4c65e64f21dac3e5db22a4c47b796905 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-27T21:35:07,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d9878bae316f4964bebe15c2b7d5a7b5_4c65e64f21dac3e5db22a4c47b796905 is 50, key is test_row_0/A:col10/1732743306035/Put/seqid=0 2024-11-27T21:35:07,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742022_1198 (size=12304) 2024-11-27T21:35:08,167 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c65e64f21dac3e5db22a4c47b796905#A#compaction#167 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:08,168 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/0c101e74219d4853a49ba63cf3b47a5d is 175, key is test_row_0/A:col10/1732743305380/Put/seqid=0 2024-11-27T21:35:08,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742023_1199 (size=31549) 2024-11-27T21:35:08,183 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/0c101e74219d4853a49ba63cf3b47a5d as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/0c101e74219d4853a49ba63cf3b47a5d 2024-11-27T21:35:08,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on 4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:08,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. as already flushing 2024-11-27T21:35:08,190 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4c65e64f21dac3e5db22a4c47b796905/A of 4c65e64f21dac3e5db22a4c47b796905 into 0c101e74219d4853a49ba63cf3b47a5d(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:08,190 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c65e64f21dac3e5db22a4c47b796905: 2024-11-27T21:35:08,190 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., storeName=4c65e64f21dac3e5db22a4c47b796905/A, priority=12, startTime=1732743307720; duration=0sec 2024-11-27T21:35:08,190 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:08,190 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c65e64f21dac3e5db22a4c47b796905:A 2024-11-27T21:35:08,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:08,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743368209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:08,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:08,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743368212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:08,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:08,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:08,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:08,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743368213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:08,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:08,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743368214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:08,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:08,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:08,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743368215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:08,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:08,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:35:08,236 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d9878bae316f4964bebe15c2b7d5a7b5_4c65e64f21dac3e5db22a4c47b796905 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d9878bae316f4964bebe15c2b7d5a7b5_4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:08,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/2da9b094e43b4e518e09672e7236020c, store: [table=TestAcidGuarantees family=A region=4c65e64f21dac3e5db22a4c47b796905] 2024-11-27T21:35:08,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/2da9b094e43b4e518e09672e7236020c is 175, key is test_row_0/A:col10/1732743306035/Put/seqid=0 2024-11-27T21:35:08,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742024_1200 (size=31105) 2024-11-27T21:35:08,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:08,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743368323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:08,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:08,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:08,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743368324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:08,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:08,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743368324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:08,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:08,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:08,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:08,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,330 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743368324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:08,330 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:08,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743368324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:08,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:08,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:08,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-27T21:35:08,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:08,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743368533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:08,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:08,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743368533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:08,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:08,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:08,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:08,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743368533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:08,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:08,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743368533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:08,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:08,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:08,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743368533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:08,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:08,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:08,642 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=214, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/2da9b094e43b4e518e09672e7236020c 2024-11-27T21:35:08,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/2b05cc9f0552404ca7ef7cd0c022bee2 is 50, key is test_row_0/B:col10/1732743306035/Put/seqid=0 2024-11-27T21:35:08,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742025_1201 (size=12151) 2024-11-27T21:35:08,667 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/2b05cc9f0552404ca7ef7cd0c022bee2 2024-11-27T21:35:08,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/1639feeceda445ce88a113b28b7942f8 is 50, key is test_row_0/C:col10/1732743306035/Put/seqid=0 2024-11-27T21:35:08,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742026_1202 (size=12151) 2024-11-27T21:35:08,680 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/1639feeceda445ce88a113b28b7942f8 2024-11-27T21:35:08,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/2da9b094e43b4e518e09672e7236020c as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/2da9b094e43b4e518e09672e7236020c 2024-11-27T21:35:08,690 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/2da9b094e43b4e518e09672e7236020c, entries=150, sequenceid=214, filesize=30.4 K 2024-11-27T21:35:08,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/2b05cc9f0552404ca7ef7cd0c022bee2 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/2b05cc9f0552404ca7ef7cd0c022bee2 2024-11-27T21:35:08,696 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/2b05cc9f0552404ca7ef7cd0c022bee2, entries=150, sequenceid=214, filesize=11.9 K 2024-11-27T21:35:08,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/1639feeceda445ce88a113b28b7942f8 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/1639feeceda445ce88a113b28b7942f8 2024-11-27T21:35:08,701 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/1639feeceda445ce88a113b28b7942f8, entries=150, sequenceid=214, filesize=11.9 K 2024-11-27T21:35:08,702 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 4c65e64f21dac3e5db22a4c47b796905 in 882ms, sequenceid=214, compaction requested=false 2024-11-27T21:35:08,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 4c65e64f21dac3e5db22a4c47b796905: 2024-11-27T21:35:08,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:08,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-27T21:35:08,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-27T21:35:08,706 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-27T21:35:08,706 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4170 sec 2024-11-27T21:35:08,709 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.4220 sec 2024-11-27T21:35:08,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on 4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:08,847 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:35:08,847 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c65e64f21dac3e5db22a4c47b796905 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-27T21:35:08,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411277345dcbc486641dfba65b6a3506e5796_4c65e64f21dac3e5db22a4c47b796905 is 50, key is test_row_0/A:col10/1732743308210/Put/seqid=0 2024-11-27T21:35:08,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742027_1203 (size=14794) 2024-11-27T21:35:08,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:08,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743368855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:08,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:08,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:08,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743368856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:08,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:08,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743368860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:08,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:08,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:08,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743368862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:08,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:08,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743368863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:08,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:08,868 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,868 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,868 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:08,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:08,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:08,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743368973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:08,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:08,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:08,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743368973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:08,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:08,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:08,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743368974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:08,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:08,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:08,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743368974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:08,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:08,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743368974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:08,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:08,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:08,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:09,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:09,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743369183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:09,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:09,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:09,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:09,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:09,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:09,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743369183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:09,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743369184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:09,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:09,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:09,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743369184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:09,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:09,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:09,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:09,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:09,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:09,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:09,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743369184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:09,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:09,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:09,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:09,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:09,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:09,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:09,261 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:35:09,266 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411277345dcbc486641dfba65b6a3506e5796_4c65e64f21dac3e5db22a4c47b796905 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411277345dcbc486641dfba65b6a3506e5796_4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:09,267 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/6a2107cabc644145a21aa53ccb043e24, store: [table=TestAcidGuarantees family=A region=4c65e64f21dac3e5db22a4c47b796905] 2024-11-27T21:35:09,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/6a2107cabc644145a21aa53ccb043e24 is 175, key is test_row_0/A:col10/1732743308210/Put/seqid=0 2024-11-27T21:35:09,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742028_1204 (size=39749) 2024-11-27T21:35:09,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:09,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743369493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:09,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:09,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:09,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:09,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:09,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:09,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743369493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:09,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743369493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:09,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:09,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:09,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:09,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:09,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743369493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:09,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:09,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743369494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:09,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:09,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:09,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:09,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:09,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:09,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:09,497 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:09,497 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:09,497 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:09,673 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=236, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/6a2107cabc644145a21aa53ccb043e24 2024-11-27T21:35:09,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/f1a68666f8c840749a618c3955345616 is 50, key is test_row_0/B:col10/1732743308210/Put/seqid=0 2024-11-27T21:35:09,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742029_1205 (size=12151) 2024-11-27T21:35:09,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/f1a68666f8c840749a618c3955345616 2024-11-27T21:35:09,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/b8ddec8c331b4eeda04b4dde3a53cdae is 50, key is test_row_0/C:col10/1732743308210/Put/seqid=0 2024-11-27T21:35:09,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742030_1206 (size=12151) 2024-11-27T21:35:10,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:10,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743370003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:10,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:10,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743370004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:10,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:10,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:10,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:10,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743370004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:10,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:10,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:10,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:10,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:10,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:10,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743370004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:10,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:10,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743370004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:10,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:10,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:10,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:10,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:10,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:10,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:10,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:10,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:10,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:10,106 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/b8ddec8c331b4eeda04b4dde3a53cdae 2024-11-27T21:35:10,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/6a2107cabc644145a21aa53ccb043e24 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/6a2107cabc644145a21aa53ccb043e24 2024-11-27T21:35:10,117 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/6a2107cabc644145a21aa53ccb043e24, entries=200, sequenceid=236, filesize=38.8 K 2024-11-27T21:35:10,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/f1a68666f8c840749a618c3955345616 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/f1a68666f8c840749a618c3955345616 2024-11-27T21:35:10,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/f1a68666f8c840749a618c3955345616, entries=150, sequenceid=236, filesize=11.9 K 2024-11-27T21:35:10,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/b8ddec8c331b4eeda04b4dde3a53cdae as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/b8ddec8c331b4eeda04b4dde3a53cdae 2024-11-27T21:35:10,140 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/b8ddec8c331b4eeda04b4dde3a53cdae, entries=150, sequenceid=236, filesize=11.9 K 2024-11-27T21:35:10,141 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 4c65e64f21dac3e5db22a4c47b796905 in 1294ms, sequenceid=236, compaction requested=true 2024-11-27T21:35:10,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c65e64f21dac3e5db22a4c47b796905: 2024-11-27T21:35:10,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c65e64f21dac3e5db22a4c47b796905:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:35:10,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:10,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c65e64f21dac3e5db22a4c47b796905:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:35:10,142 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:35:10,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:35:10,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c65e64f21dac3e5db22a4c47b796905:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:35:10,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-27T21:35:10,142 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:35:10,144 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:35:10,144 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): 4c65e64f21dac3e5db22a4c47b796905/A is initiating minor compaction (all files) 2024-11-27T21:35:10,144 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c65e64f21dac3e5db22a4c47b796905/A in TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:10,144 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/0c101e74219d4853a49ba63cf3b47a5d, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/2da9b094e43b4e518e09672e7236020c, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/6a2107cabc644145a21aa53ccb043e24] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp, totalSize=100.0 K 2024-11-27T21:35:10,144 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:10,144 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. files: [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/0c101e74219d4853a49ba63cf3b47a5d, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/2da9b094e43b4e518e09672e7236020c, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/6a2107cabc644145a21aa53ccb043e24] 2024-11-27T21:35:10,145 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:35:10,145 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): 4c65e64f21dac3e5db22a4c47b796905/B is initiating minor compaction (all files) 2024-11-27T21:35:10,145 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c65e64f21dac3e5db22a4c47b796905/B in TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:10,145 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/eac5727c995542eea7b66a925ae703e3, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/2b05cc9f0552404ca7ef7cd0c022bee2, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/f1a68666f8c840749a618c3955345616] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp, totalSize=36.0 K 2024-11-27T21:35:10,146 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting eac5727c995542eea7b66a925ae703e3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732743305377 2024-11-27T21:35:10,146 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c101e74219d4853a49ba63cf3b47a5d, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732743305377 2024-11-27T21:35:10,146 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b05cc9f0552404ca7ef7cd0c022bee2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732743306029 2024-11-27T21:35:10,146 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 2da9b094e43b4e518e09672e7236020c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732743306029 2024-11-27T21:35:10,146 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1a68666f8c840749a618c3955345616, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732743308206 2024-11-27T21:35:10,147 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a2107cabc644145a21aa53ccb043e24, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732743308206 2024-11-27T21:35:10,155 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4c65e64f21dac3e5db22a4c47b796905] 2024-11-27T21:35:10,158 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127248c2f65f92642a4a34c811a4ced8406_4c65e64f21dac3e5db22a4c47b796905 store=[table=TestAcidGuarantees family=A region=4c65e64f21dac3e5db22a4c47b796905] 2024-11-27T21:35:10,158 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c65e64f21dac3e5db22a4c47b796905#B#compaction#176 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:10,159 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/3e8c5cf4edd84752b8ca33d56836d8a0 is 50, key is test_row_0/B:col10/1732743308210/Put/seqid=0 2024-11-27T21:35:10,160 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127248c2f65f92642a4a34c811a4ced8406_4c65e64f21dac3e5db22a4c47b796905, store=[table=TestAcidGuarantees family=A region=4c65e64f21dac3e5db22a4c47b796905] 2024-11-27T21:35:10,160 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127248c2f65f92642a4a34c811a4ced8406_4c65e64f21dac3e5db22a4c47b796905 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4c65e64f21dac3e5db22a4c47b796905] 2024-11-27T21:35:10,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742032_1208 (size=4469) 2024-11-27T21:35:10,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742031_1207 (size=12697) 2024-11-27T21:35:10,184 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c65e64f21dac3e5db22a4c47b796905#A#compaction#175 average throughput is 0.84 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:10,185 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/2ea03a414f094f2395de3dc71625b7cb is 175, key is test_row_0/A:col10/1732743308210/Put/seqid=0 2024-11-27T21:35:10,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742033_1209 (size=31651) 2024-11-27T21:35:10,193 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/3e8c5cf4edd84752b8ca33d56836d8a0 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/3e8c5cf4edd84752b8ca33d56836d8a0 2024-11-27T21:35:10,199 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c65e64f21dac3e5db22a4c47b796905/B of 4c65e64f21dac3e5db22a4c47b796905 into 3e8c5cf4edd84752b8ca33d56836d8a0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:10,199 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c65e64f21dac3e5db22a4c47b796905: 2024-11-27T21:35:10,199 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., storeName=4c65e64f21dac3e5db22a4c47b796905/B, priority=13, startTime=1732743310142; duration=0sec 2024-11-27T21:35:10,199 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:35:10,199 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c65e64f21dac3e5db22a4c47b796905:B 2024-11-27T21:35:10,199 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:35:10,200 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:35:10,200 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): 4c65e64f21dac3e5db22a4c47b796905/C is initiating minor compaction (all files) 2024-11-27T21:35:10,200 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c65e64f21dac3e5db22a4c47b796905/C in TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:10,200 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/849bb24a746146e1883b3e335fbd7dc6, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/1639feeceda445ce88a113b28b7942f8, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/b8ddec8c331b4eeda04b4dde3a53cdae] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp, totalSize=36.0 K 2024-11-27T21:35:10,201 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 849bb24a746146e1883b3e335fbd7dc6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732743305377 2024-11-27T21:35:10,201 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1639feeceda445ce88a113b28b7942f8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732743306029 2024-11-27T21:35:10,201 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8ddec8c331b4eeda04b4dde3a53cdae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732743308206 2024-11-27T21:35:10,210 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c65e64f21dac3e5db22a4c47b796905#C#compaction#177 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:10,211 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/07415fc793644c33b863bc326611a357 is 50, key is test_row_0/C:col10/1732743308210/Put/seqid=0 2024-11-27T21:35:10,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742034_1210 (size=12663) 2024-11-27T21:35:10,229 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/07415fc793644c33b863bc326611a357 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/07415fc793644c33b863bc326611a357 2024-11-27T21:35:10,236 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c65e64f21dac3e5db22a4c47b796905/C of 4c65e64f21dac3e5db22a4c47b796905 into 07415fc793644c33b863bc326611a357(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:10,236 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c65e64f21dac3e5db22a4c47b796905: 2024-11-27T21:35:10,236 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., storeName=4c65e64f21dac3e5db22a4c47b796905/C, priority=13, startTime=1732743310142; duration=0sec 2024-11-27T21:35:10,236 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:10,236 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c65e64f21dac3e5db22a4c47b796905:C 2024-11-27T21:35:10,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-27T21:35:10,454 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:35:10,454 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:35:10,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-27T21:35:10,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-27T21:35:10,457 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:35:10,458 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:35:10,458 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:35:10,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-27T21:35:10,590 INFO [master/4585c39bed03:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-27T21:35:10,590 INFO [master/4585c39bed03:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-27T21:35:10,597 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/2ea03a414f094f2395de3dc71625b7cb as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/2ea03a414f094f2395de3dc71625b7cb 2024-11-27T21:35:10,607 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c65e64f21dac3e5db22a4c47b796905/A of 4c65e64f21dac3e5db22a4c47b796905 into 2ea03a414f094f2395de3dc71625b7cb(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:10,607 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c65e64f21dac3e5db22a4c47b796905: 2024-11-27T21:35:10,607 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., storeName=4c65e64f21dac3e5db22a4c47b796905/A, priority=13, startTime=1732743310142; duration=0sec 2024-11-27T21:35:10,607 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:10,607 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c65e64f21dac3e5db22a4c47b796905:A 2024-11-27T21:35:10,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-27T21:35:10,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:10,611 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 4c65e64f21dac3e5db22a4c47b796905 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-27T21:35:10,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112711a9966a241d45b1a7f5576ffc403afe_4c65e64f21dac3e5db22a4c47b796905 is 50, key is test_row_0/A:col10/1732743308860/Put/seqid=0 2024-11-27T21:35:10,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742035_1211 (size=12304) 2024-11-27T21:35:10,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:35:10,635 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112711a9966a241d45b1a7f5576ffc403afe_4c65e64f21dac3e5db22a4c47b796905 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112711a9966a241d45b1a7f5576ffc403afe_4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:10,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/e3665866464840429d41e096d700d0cb, store: [table=TestAcidGuarantees family=A region=4c65e64f21dac3e5db22a4c47b796905] 2024-11-27T21:35:10,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/e3665866464840429d41e096d700d0cb is 175, key is test_row_0/A:col10/1732743308860/Put/seqid=0 2024-11-27T21:35:10,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742036_1212 (size=31105) 2024-11-27T21:35:10,642 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=254, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/e3665866464840429d41e096d700d0cb 2024-11-27T21:35:10,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/4f617b5d64c34721a31c5bbefd53ca99 is 50, key is test_row_0/B:col10/1732743308860/Put/seqid=0 2024-11-27T21:35:10,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742037_1213 (size=12151) 2024-11-27T21:35:10,665 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/4f617b5d64c34721a31c5bbefd53ca99 2024-11-27T21:35:10,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/cef0ee1969c84b34884f2a000b0d1278 is 50, key is test_row_0/C:col10/1732743308860/Put/seqid=0 2024-11-27T21:35:10,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742038_1214 (size=12151) 2024-11-27T21:35:10,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-27T21:35:11,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on 4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:11,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. as already flushing 2024-11-27T21:35:11,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:11,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743371064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:11,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:11,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:11,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743371065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:11,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743371067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:11,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:11,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:11,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743371068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:11,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:11,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:11,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743371069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:11,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:11,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:11,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:11,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-27T21:35:11,084 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/cef0ee1969c84b34884f2a000b0d1278 2024-11-27T21:35:11,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/e3665866464840429d41e096d700d0cb as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/e3665866464840429d41e096d700d0cb 2024-11-27T21:35:11,094 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/e3665866464840429d41e096d700d0cb, entries=150, sequenceid=254, filesize=30.4 K 2024-11-27T21:35:11,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/4f617b5d64c34721a31c5bbefd53ca99 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/4f617b5d64c34721a31c5bbefd53ca99 2024-11-27T21:35:11,102 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/4f617b5d64c34721a31c5bbefd53ca99, entries=150, sequenceid=254, filesize=11.9 K 2024-11-27T21:35:11,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/cef0ee1969c84b34884f2a000b0d1278 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/cef0ee1969c84b34884f2a000b0d1278 2024-11-27T21:35:11,108 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/cef0ee1969c84b34884f2a000b0d1278, entries=150, sequenceid=254, filesize=11.9 K 2024-11-27T21:35:11,110 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 4c65e64f21dac3e5db22a4c47b796905 in 498ms, sequenceid=254, compaction requested=false 2024-11-27T21:35:11,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 4c65e64f21dac3e5db22a4c47b796905: 2024-11-27T21:35:11,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:11,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-27T21:35:11,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-27T21:35:11,113 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-27T21:35:11,113 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 653 msec 2024-11-27T21:35:11,116 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 660 msec 2024-11-27T21:35:11,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on 4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:11,187 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:35:11,187 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c65e64f21dac3e5db22a4c47b796905 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-27T21:35:11,197 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112762c9e887d0774ed3b556da4f2f0bd1e6_4c65e64f21dac3e5db22a4c47b796905 is 50, key is test_row_0/A:col10/1732743311062/Put/seqid=0 2024-11-27T21:35:11,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:11,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743371198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:11,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:11,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743371199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:11,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:11,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:11,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742039_1215 (size=17534) 2024-11-27T21:35:11,204 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:35:11,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:11,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743371201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:11,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:11,209 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112762c9e887d0774ed3b556da4f2f0bd1e6_4c65e64f21dac3e5db22a4c47b796905 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112762c9e887d0774ed3b556da4f2f0bd1e6_4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:11,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743371202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:11,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:11,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:11,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:11,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743371203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:11,210 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,210 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,210 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:11,211 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/a04e3d5113d34138b09f6ae855f7857b, store: [table=TestAcidGuarantees family=A region=4c65e64f21dac3e5db22a4c47b796905] 2024-11-27T21:35:11,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/a04e3d5113d34138b09f6ae855f7857b is 175, key is test_row_0/A:col10/1732743311062/Put/seqid=0 2024-11-27T21:35:11,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742040_1216 (size=48639) 2024-11-27T21:35:11,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:11,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743371304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:11,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:11,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:11,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743371313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:11,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:11,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:11,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743371313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:11,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:11,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:11,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743371313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:11,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:11,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:11,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743371313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:11,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:11,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:11,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743371514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:11,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:11,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:11,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:11,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743371523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:11,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743371523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:11,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:11,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743371523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:11,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:11,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:11,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:11,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743371523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:11,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:11,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:11,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-27T21:35:11,593 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:35:11,594 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:35:11,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-27T21:35:11,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-27T21:35:11,596 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:35:11,597 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:35:11,597 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:35:11,619 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=280, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/a04e3d5113d34138b09f6ae855f7857b 2024-11-27T21:35:11,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/df03c05d59e246b6ad4274d7c8b2cd41 is 50, key is test_row_0/B:col10/1732743311062/Put/seqid=0 2024-11-27T21:35:11,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742041_1217 (size=12301) 2024-11-27T21:35:11,637 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/df03c05d59e246b6ad4274d7c8b2cd41 2024-11-27T21:35:11,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/07399186f1814de882ff28ce63a2ea53 is 50, key is test_row_0/C:col10/1732743311062/Put/seqid=0 2024-11-27T21:35:11,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742042_1218 (size=12301) 2024-11-27T21:35:11,652 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/07399186f1814de882ff28ce63a2ea53 2024-11-27T21:35:11,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/a04e3d5113d34138b09f6ae855f7857b as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/a04e3d5113d34138b09f6ae855f7857b 2024-11-27T21:35:11,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/a04e3d5113d34138b09f6ae855f7857b, entries=250, sequenceid=280, filesize=47.5 K 2024-11-27T21:35:11,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/df03c05d59e246b6ad4274d7c8b2cd41 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/df03c05d59e246b6ad4274d7c8b2cd41 2024-11-27T21:35:11,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/df03c05d59e246b6ad4274d7c8b2cd41, entries=150, sequenceid=280, filesize=12.0 K 2024-11-27T21:35:11,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/07399186f1814de882ff28ce63a2ea53 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/07399186f1814de882ff28ce63a2ea53 2024-11-27T21:35:11,673 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/07399186f1814de882ff28ce63a2ea53, entries=150, sequenceid=280, filesize=12.0 K 2024-11-27T21:35:11,675 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 4c65e64f21dac3e5db22a4c47b796905 in 487ms, sequenceid=280, compaction requested=true 2024-11-27T21:35:11,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c65e64f21dac3e5db22a4c47b796905: 2024-11-27T21:35:11,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c65e64f21dac3e5db22a4c47b796905:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:35:11,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:11,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c65e64f21dac3e5db22a4c47b796905:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:35:11,675 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:35:11,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:11,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c65e64f21dac3e5db22a4c47b796905:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:35:11,675 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:35:11,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:35:11,676 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111395 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:35:11,676 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:35:11,676 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): 4c65e64f21dac3e5db22a4c47b796905/A is initiating minor compaction (all files) 2024-11-27T21:35:11,676 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): 4c65e64f21dac3e5db22a4c47b796905/B is initiating minor compaction (all files) 2024-11-27T21:35:11,676 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c65e64f21dac3e5db22a4c47b796905/A in TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:11,676 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c65e64f21dac3e5db22a4c47b796905/B in TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:11,676 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/2ea03a414f094f2395de3dc71625b7cb, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/e3665866464840429d41e096d700d0cb, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/a04e3d5113d34138b09f6ae855f7857b] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp, totalSize=108.8 K 2024-11-27T21:35:11,676 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:11,676 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/3e8c5cf4edd84752b8ca33d56836d8a0, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/4f617b5d64c34721a31c5bbefd53ca99, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/df03c05d59e246b6ad4274d7c8b2cd41] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp, totalSize=36.3 K 2024-11-27T21:35:11,676 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. files: [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/2ea03a414f094f2395de3dc71625b7cb, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/e3665866464840429d41e096d700d0cb, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/a04e3d5113d34138b09f6ae855f7857b] 2024-11-27T21:35:11,677 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ea03a414f094f2395de3dc71625b7cb, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732743308206 2024-11-27T21:35:11,677 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e8c5cf4edd84752b8ca33d56836d8a0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732743308206 2024-11-27T21:35:11,677 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3665866464840429d41e096d700d0cb, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732743308859 2024-11-27T21:35:11,677 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f617b5d64c34721a31c5bbefd53ca99, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732743308859 2024-11-27T21:35:11,677 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting a04e3d5113d34138b09f6ae855f7857b, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732743311062 2024-11-27T21:35:11,677 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting df03c05d59e246b6ad4274d7c8b2cd41, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732743311062 2024-11-27T21:35:11,685 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4c65e64f21dac3e5db22a4c47b796905] 2024-11-27T21:35:11,686 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c65e64f21dac3e5db22a4c47b796905#B#compaction#185 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:11,686 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/a2d323107c52477e96543f9cfde59e4a is 50, key is test_row_0/B:col10/1732743311062/Put/seqid=0 2024-11-27T21:35:11,693 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127b0962e66ec6a4e318ee62f79e4aace68_4c65e64f21dac3e5db22a4c47b796905 store=[table=TestAcidGuarantees family=A region=4c65e64f21dac3e5db22a4c47b796905] 2024-11-27T21:35:11,695 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127b0962e66ec6a4e318ee62f79e4aace68_4c65e64f21dac3e5db22a4c47b796905, store=[table=TestAcidGuarantees family=A region=4c65e64f21dac3e5db22a4c47b796905] 2024-11-27T21:35:11,695 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127b0962e66ec6a4e318ee62f79e4aace68_4c65e64f21dac3e5db22a4c47b796905 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4c65e64f21dac3e5db22a4c47b796905] 2024-11-27T21:35:11,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742043_1219 (size=12949) 2024-11-27T21:35:11,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742044_1220 (size=4469) 2024-11-27T21:35:11,702 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c65e64f21dac3e5db22a4c47b796905#A#compaction#184 average throughput is 1.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:11,702 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/5375099c97c84ba2843abffc237e7bf4 is 175, key is test_row_0/A:col10/1732743311062/Put/seqid=0 2024-11-27T21:35:11,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-27T21:35:11,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742045_1221 (size=31903) 2024-11-27T21:35:11,707 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/a2d323107c52477e96543f9cfde59e4a as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/a2d323107c52477e96543f9cfde59e4a 2024-11-27T21:35:11,716 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/5375099c97c84ba2843abffc237e7bf4 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/5375099c97c84ba2843abffc237e7bf4 2024-11-27T21:35:11,720 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c65e64f21dac3e5db22a4c47b796905/B of 4c65e64f21dac3e5db22a4c47b796905 into a2d323107c52477e96543f9cfde59e4a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:11,720 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c65e64f21dac3e5db22a4c47b796905: 2024-11-27T21:35:11,720 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., storeName=4c65e64f21dac3e5db22a4c47b796905/B, priority=13, startTime=1732743311675; duration=0sec 2024-11-27T21:35:11,720 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:35:11,720 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c65e64f21dac3e5db22a4c47b796905:B 2024-11-27T21:35:11,720 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:35:11,722 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:35:11,722 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): 4c65e64f21dac3e5db22a4c47b796905/C is initiating minor compaction (all files) 2024-11-27T21:35:11,722 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c65e64f21dac3e5db22a4c47b796905/C in TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:11,722 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/07415fc793644c33b863bc326611a357, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/cef0ee1969c84b34884f2a000b0d1278, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/07399186f1814de882ff28ce63a2ea53] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp, totalSize=36.2 K 2024-11-27T21:35:11,722 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 07415fc793644c33b863bc326611a357, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732743308206 2024-11-27T21:35:11,723 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting cef0ee1969c84b34884f2a000b0d1278, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732743308859 2024-11-27T21:35:11,723 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c65e64f21dac3e5db22a4c47b796905/A of 4c65e64f21dac3e5db22a4c47b796905 into 5375099c97c84ba2843abffc237e7bf4(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:11,723 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c65e64f21dac3e5db22a4c47b796905: 2024-11-27T21:35:11,723 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., storeName=4c65e64f21dac3e5db22a4c47b796905/A, priority=13, startTime=1732743311675; duration=0sec 2024-11-27T21:35:11,723 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 07399186f1814de882ff28ce63a2ea53, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732743311062 2024-11-27T21:35:11,723 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:11,724 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c65e64f21dac3e5db22a4c47b796905:A 2024-11-27T21:35:11,733 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c65e64f21dac3e5db22a4c47b796905#C#compaction#186 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:11,733 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/74dfc12f76c94f08be9c1890cc5268aa is 50, key is test_row_0/C:col10/1732743311062/Put/seqid=0 2024-11-27T21:35:11,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742046_1222 (size=12915) 2024-11-27T21:35:11,744 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/74dfc12f76c94f08be9c1890cc5268aa as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/74dfc12f76c94f08be9c1890cc5268aa 2024-11-27T21:35:11,750 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c65e64f21dac3e5db22a4c47b796905/C of 4c65e64f21dac3e5db22a4c47b796905 into 74dfc12f76c94f08be9c1890cc5268aa(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:11,750 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c65e64f21dac3e5db22a4c47b796905: 2024-11-27T21:35:11,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-27T21:35:11,750 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., storeName=4c65e64f21dac3e5db22a4c47b796905/C, priority=13, startTime=1732743311675; duration=0sec 2024-11-27T21:35:11,750 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:11,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:11,750 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c65e64f21dac3e5db22a4c47b796905:C 2024-11-27T21:35:11,750 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 4c65e64f21dac3e5db22a4c47b796905 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-27T21:35:11,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127bda8b8f7aad64077a2e3089f25ec5d18_4c65e64f21dac3e5db22a4c47b796905 is 50, key is test_row_0/A:col10/1732743311200/Put/seqid=0 2024-11-27T21:35:11,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742047_1223 (size=12454) 2024-11-27T21:35:11,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on 4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:11,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. as already flushing 2024-11-27T21:35:11,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:11,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743371868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:11,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:11,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743371869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:11,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:11,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:11,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743371870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:11,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:11,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:11,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:11,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743371871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:11,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:11,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:11,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743371872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:11,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:11,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-27T21:35:11,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:11,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743371983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:11,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:11,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:11,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743371984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:11,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:11,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:11,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743371984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:11,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:11,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:11,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743371984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:11,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:11,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743371984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:11,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:11,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:11,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:12,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:35:12,166 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127bda8b8f7aad64077a2e3089f25ec5d18_4c65e64f21dac3e5db22a4c47b796905 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127bda8b8f7aad64077a2e3089f25ec5d18_4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:12,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/d4c514cd977649f3828d2d11cc1c83c3, store: [table=TestAcidGuarantees family=A region=4c65e64f21dac3e5db22a4c47b796905] 2024-11-27T21:35:12,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/d4c514cd977649f3828d2d11cc1c83c3 is 175, key is test_row_0/A:col10/1732743311200/Put/seqid=0 2024-11-27T21:35:12,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742048_1224 (size=31255) 2024-11-27T21:35:12,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:12,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743372194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:12,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:12,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:12,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:12,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:12,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743372194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:12,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:12,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:12,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743372194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:12,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:12,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:12,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:12,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:12,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:12,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:12,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743372194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:12,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:12,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:12,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:12,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:12,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743372194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:12,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:12,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:12,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:12,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-27T21:35:12,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:12,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743372503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:12,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:12,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:12,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:12,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743372504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:12,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:12,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:12,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:12,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743372504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:12,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:12,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:12,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:12,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743372504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:12,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:12,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743372504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:12,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:12,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:12,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:12,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:12,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:12,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:12,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:12,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:12,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:12,573 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=293, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/d4c514cd977649f3828d2d11cc1c83c3 2024-11-27T21:35:12,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/28c854374d624fc2b0b67ea7e5db2bc2 is 50, key is test_row_0/B:col10/1732743311200/Put/seqid=0 2024-11-27T21:35:12,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742049_1225 (size=12301) 2024-11-27T21:35:12,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-27T21:35:12,986 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/28c854374d624fc2b0b67ea7e5db2bc2 2024-11-27T21:35:12,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/3e6ae27a754244f185d79f3d9fcece65 is 50, key is test_row_0/C:col10/1732743311200/Put/seqid=0 2024-11-27T21:35:12,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742050_1226 (size=12301) 2024-11-27T21:35:13,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:13,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743373013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:13,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:13,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:13,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:13,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:13,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:13,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743373013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:13,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743373013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:13,018 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:13,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:13,018 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:13,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:13,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743373013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:13,018 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:13,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:13,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:13,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:13,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743373013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:13,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:13,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:13,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:13,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:13,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:13,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:13,400 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/3e6ae27a754244f185d79f3d9fcece65 2024-11-27T21:35:13,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/d4c514cd977649f3828d2d11cc1c83c3 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/d4c514cd977649f3828d2d11cc1c83c3 2024-11-27T21:35:13,410 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/d4c514cd977649f3828d2d11cc1c83c3, entries=150, sequenceid=293, filesize=30.5 K 2024-11-27T21:35:13,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/28c854374d624fc2b0b67ea7e5db2bc2 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/28c854374d624fc2b0b67ea7e5db2bc2 2024-11-27T21:35:13,416 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/28c854374d624fc2b0b67ea7e5db2bc2, entries=150, sequenceid=293, filesize=12.0 K 2024-11-27T21:35:13,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/3e6ae27a754244f185d79f3d9fcece65 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/3e6ae27a754244f185d79f3d9fcece65 2024-11-27T21:35:13,422 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/3e6ae27a754244f185d79f3d9fcece65, entries=150, sequenceid=293, filesize=12.0 K 2024-11-27T21:35:13,423 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 4c65e64f21dac3e5db22a4c47b796905 in 1673ms, sequenceid=293, compaction requested=false 2024-11-27T21:35:13,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 4c65e64f21dac3e5db22a4c47b796905: 2024-11-27T21:35:13,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:13,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-27T21:35:13,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-27T21:35:13,427 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-27T21:35:13,427 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8280 sec 2024-11-27T21:35:13,429 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.8340 sec 2024-11-27T21:35:13,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-27T21:35:13,744 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:35:13,744 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:35:13,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-27T21:35:13,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-27T21:35:13,746 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:35:13,747 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:35:13,747 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:35:13,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-27T21:35:13,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-27T21:35:13,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:13,900 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 4c65e64f21dac3e5db22a4c47b796905 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-27T21:35:13,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112702e9001262184961bc490af6495c4e6c_4c65e64f21dac3e5db22a4c47b796905 is 50, key is test_row_0/A:col10/1732743311871/Put/seqid=0 2024-11-27T21:35:13,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742051_1227 (size=12454) 2024-11-27T21:35:14,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on 4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:14,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. as already flushing 2024-11-27T21:35:14,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:14,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743374033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:14,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:14,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743374033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:14,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:14,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743374033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:14,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:14,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:14,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:14,038 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,39097,1732743267048 for 'test_row_2' in 4c65e64f21dac3e5db22a4c47b796905 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:35:14,038 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,39097,1732743267048 for 'test_row_0' in 4c65e64f21dac3e5db22a4c47b796905 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:35:14,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:14,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:14,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:14,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:14,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:14,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:14,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:14,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743374036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:14,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:14,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:14,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:14,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:14,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743374037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:14,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:14,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:14,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:14,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-27T21:35:14,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:14,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743374143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:14,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:14,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:14,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:14,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743374143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:14,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:14,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:14,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:14,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:14,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:14,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743374154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:14,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:14,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:14,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:14,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:35:14,316 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112702e9001262184961bc490af6495c4e6c_4c65e64f21dac3e5db22a4c47b796905 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112702e9001262184961bc490af6495c4e6c_4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:14,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/e126b4cb12b7446d83b753ce7a0998a7, store: [table=TestAcidGuarantees family=A region=4c65e64f21dac3e5db22a4c47b796905] 2024-11-27T21:35:14,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/e126b4cb12b7446d83b753ce7a0998a7 is 175, key is test_row_0/A:col10/1732743311871/Put/seqid=0 2024-11-27T21:35:14,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742052_1228 (size=31255) 2024-11-27T21:35:14,322 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=319, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/e126b4cb12b7446d83b753ce7a0998a7 2024-11-27T21:35:14,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/8eb57d46a85b4e5d95f9ef6a11f41338 is 50, key is test_row_0/B:col10/1732743311871/Put/seqid=0 2024-11-27T21:35:14,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742053_1229 (size=12301) 2024-11-27T21:35:14,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:14,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743374353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:14,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:14,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:14,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:14,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:14,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743374353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:14,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:14,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:14,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:14,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:14,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743374363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:14,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:14,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:14,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:14,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-27T21:35:14,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:14,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743374664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:14,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:14,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:14,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:14,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:14,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743374664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:14,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:14,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:14,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:14,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:14,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743374673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:14,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:14,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:14,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:14,735 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/8eb57d46a85b4e5d95f9ef6a11f41338 2024-11-27T21:35:14,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/af4c168809d843c0bf938187dde57acd is 50, key is test_row_0/C:col10/1732743311871/Put/seqid=0 2024-11-27T21:35:14,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742054_1230 (size=12301) 2024-11-27T21:35:14,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-27T21:35:15,149 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/af4c168809d843c0bf938187dde57acd 2024-11-27T21:35:15,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/e126b4cb12b7446d83b753ce7a0998a7 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/e126b4cb12b7446d83b753ce7a0998a7 2024-11-27T21:35:15,159 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/e126b4cb12b7446d83b753ce7a0998a7, entries=150, sequenceid=319, filesize=30.5 K 2024-11-27T21:35:15,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/8eb57d46a85b4e5d95f9ef6a11f41338 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/8eb57d46a85b4e5d95f9ef6a11f41338 2024-11-27T21:35:15,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:15,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743375176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:15,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:15,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:15,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:15,181 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/8eb57d46a85b4e5d95f9ef6a11f41338, entries=150, sequenceid=319, filesize=12.0 K 2024-11-27T21:35:15,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/af4c168809d843c0bf938187dde57acd as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/af4c168809d843c0bf938187dde57acd 2024-11-27T21:35:15,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:15,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743375183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:15,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:15,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:15,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:15,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743375183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:15,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:15,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:15,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:15,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:15,187 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/af4c168809d843c0bf938187dde57acd, entries=150, sequenceid=319, filesize=12.0 K 2024-11-27T21:35:15,189 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 4c65e64f21dac3e5db22a4c47b796905 in 1289ms, sequenceid=319, compaction requested=true 2024-11-27T21:35:15,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 4c65e64f21dac3e5db22a4c47b796905: 2024-11-27T21:35:15,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:15,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-27T21:35:15,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-27T21:35:15,193 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-27T21:35:15,193 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4440 sec 2024-11-27T21:35:15,195 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.4500 sec 2024-11-27T21:35:15,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-27T21:35:15,893 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:35:15,894 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:35:15,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-27T21:35:15,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-27T21:35:15,896 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:35:15,897 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:35:15,897 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:35:16,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-27T21:35:16,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-27T21:35:16,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:16,049 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 4c65e64f21dac3e5db22a4c47b796905 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-27T21:35:16,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on 4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:16,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. as already flushing 2024-11-27T21:35:16,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d0bdeab3d2b84029a5c444c82065890e_4c65e64f21dac3e5db22a4c47b796905 is 50, key is test_row_0/A:col10/1732743314032/Put/seqid=0 2024-11-27T21:35:16,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742055_1231 (size=12454) 2024-11-27T21:35:16,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:16,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743376107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:16,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:16,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:16,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:16,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:16,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743376109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:16,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:16,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:16,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:16,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:16,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743376194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:16,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:16,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743376194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:16,197 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,39097,1732743267048 for 'test_row_0' in 4c65e64f21dac3e5db22a4c47b796905 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:35:16,197 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,39097,1732743267048 for 'test_row_2' in 4c65e64f21dac3e5db22a4c47b796905 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:35:16,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:16,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:16,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:16,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:16,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:16,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:16,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:16,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743376203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:16,205 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,39097,1732743267048 for 'test_row_0' in 4c65e64f21dac3e5db22a4c47b796905 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:35:16,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:16,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:16,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:16,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-27T21:35:16,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:16,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:16,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743376223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:16,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743376223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:16,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:16,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:16,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:16,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:16,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:16,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:16,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:16,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743376433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:16,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:16,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:16,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:16,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:16,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743376433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:16,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:16,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:16,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:16,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:35:16,465 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d0bdeab3d2b84029a5c444c82065890e_4c65e64f21dac3e5db22a4c47b796905 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d0bdeab3d2b84029a5c444c82065890e_4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:16,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/ae1e0f6aedb04927a2e17a3f7e64d5cf, store: [table=TestAcidGuarantees family=A region=4c65e64f21dac3e5db22a4c47b796905] 2024-11-27T21:35:16,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/ae1e0f6aedb04927a2e17a3f7e64d5cf is 175, key is test_row_0/A:col10/1732743314032/Put/seqid=0 2024-11-27T21:35:16,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742056_1232 (size=31255) 2024-11-27T21:35:16,477 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=329, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/ae1e0f6aedb04927a2e17a3f7e64d5cf 2024-11-27T21:35:16,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/8fd69e615af54ccba80645077664f82f is 50, key is test_row_0/B:col10/1732743314032/Put/seqid=0 2024-11-27T21:35:16,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742057_1233 (size=12301) 2024-11-27T21:35:16,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-27T21:35:16,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:16,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743376743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:16,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:16,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:16,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:16,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:16,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743376753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:16,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:16,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:16,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:16,890 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/8fd69e615af54ccba80645077664f82f 2024-11-27T21:35:16,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/dfca400feb97498da55ae585fa5e5421 is 50, key is test_row_0/C:col10/1732743314032/Put/seqid=0 2024-11-27T21:35:16,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742058_1234 (size=12301) 2024-11-27T21:35:17,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-27T21:35:17,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:17,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743377254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:17,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:17,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:17,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:17,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:17,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743377263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:17,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:17,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:17,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:17,303 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/dfca400feb97498da55ae585fa5e5421 2024-11-27T21:35:17,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/ae1e0f6aedb04927a2e17a3f7e64d5cf as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/ae1e0f6aedb04927a2e17a3f7e64d5cf 2024-11-27T21:35:17,313 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/ae1e0f6aedb04927a2e17a3f7e64d5cf, entries=150, sequenceid=329, filesize=30.5 K 2024-11-27T21:35:17,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/8fd69e615af54ccba80645077664f82f as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/8fd69e615af54ccba80645077664f82f 2024-11-27T21:35:17,318 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/8fd69e615af54ccba80645077664f82f, entries=150, sequenceid=329, filesize=12.0 K 2024-11-27T21:35:17,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/dfca400feb97498da55ae585fa5e5421 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/dfca400feb97498da55ae585fa5e5421 2024-11-27T21:35:17,324 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/dfca400feb97498da55ae585fa5e5421, entries=150, sequenceid=329, filesize=12.0 K 2024-11-27T21:35:17,324 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 4c65e64f21dac3e5db22a4c47b796905 in 1275ms, sequenceid=329, compaction requested=true 2024-11-27T21:35:17,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 4c65e64f21dac3e5db22a4c47b796905: 2024-11-27T21:35:17,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:17,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-27T21:35:17,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-27T21:35:17,328 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-27T21:35:17,328 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4290 sec 2024-11-27T21:35:17,330 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 1.4350 sec 2024-11-27T21:35:18,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-27T21:35:18,054 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:35:18,054 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:35:18,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-27T21:35:18,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-27T21:35:18,071 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:35:18,072 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:35:18,072 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:35:18,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-27T21:35:18,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on 4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:18,214 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:35:18,214 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c65e64f21dac3e5db22a4c47b796905 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-27T21:35:18,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127f34da35d48584c52bdf4b8a7bf557531_4c65e64f21dac3e5db22a4c47b796905 is 50, key is test_row_0/A:col10/1732743318213/Put/seqid=0 2024-11-27T21:35:18,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-27T21:35:18,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:18,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. as already flushing 2024-11-27T21:35:18,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:18,225 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:18,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:18,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:18,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742059_1235 (size=17534) 2024-11-27T21:35:18,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:18,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743378226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:18,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:18,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:18,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743378227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:18,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:18,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:18,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:18,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:18,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:18,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:18,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41676 deadline: 1732743378233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:18,236 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,39097,1732743267048 for 'test_row_0' in 4c65e64f21dac3e5db22a4c47b796905 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:35:18,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:18,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:18,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:18,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:18,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41696 deadline: 1732743378264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:18,264 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,39097,1732743267048 for 'test_row_1' in 4c65e64f21dac3e5db22a4c47b796905 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:35:18,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:18,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:18,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:18,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:18,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732743378273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:18,276 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,39097,1732743267048 for 'test_row_1' in 4c65e64f21dac3e5db22a4c47b796905 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:35:18,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:18,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:18,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:18,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:18,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743378343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:18,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:18,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:18,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:18,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:18,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743378343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:18,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:18,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:18,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:18,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-27T21:35:18,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:18,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. as already flushing 2024-11-27T21:35:18,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:18,378 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:18,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:18,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:18,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-27T21:35:18,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-27T21:35:18,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:18,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. as already flushing 2024-11-27T21:35:18,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:18,531 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:18,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:18,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:18,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:18,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743378553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:18,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:18,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:18,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:18,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:18,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743378553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:18,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:18,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:18,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:18,630 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:35:18,634 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127f34da35d48584c52bdf4b8a7bf557531_4c65e64f21dac3e5db22a4c47b796905 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f34da35d48584c52bdf4b8a7bf557531_4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:18,635 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/ebf6796361a44af38f70e11f982dda69, store: [table=TestAcidGuarantees family=A region=4c65e64f21dac3e5db22a4c47b796905] 2024-11-27T21:35:18,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/ebf6796361a44af38f70e11f982dda69 is 175, key is test_row_0/A:col10/1732743318213/Put/seqid=0 2024-11-27T21:35:18,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742060_1236 (size=48639) 2024-11-27T21:35:18,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-27T21:35:18,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:18,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. as already flushing 2024-11-27T21:35:18,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:18,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:18,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:18,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:18,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-27T21:35:18,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-27T21:35:18,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:18,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. as already flushing 2024-11-27T21:35:18,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:18,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:18,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:18,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:18,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:18,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743378863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:18,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:18,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:18,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:18,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:18,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743378863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:18,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:18,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:18,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:18,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-27T21:35:18,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:18,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. as already flushing 2024-11-27T21:35:18,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:18,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:18,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:18,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:19,041 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=357, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/ebf6796361a44af38f70e11f982dda69 2024-11-27T21:35:19,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/8afc08bd82764c649ff582976f074f73 is 50, key is test_row_0/B:col10/1732743318213/Put/seqid=0 2024-11-27T21:35:19,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742061_1237 (size=12301) 2024-11-27T21:35:19,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-27T21:35:19,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:19,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. as already flushing 2024-11-27T21:35:19,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:19,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:19,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:19,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:19,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-27T21:35:19,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-27T21:35:19,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:19,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. as already flushing 2024-11-27T21:35:19,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:19,296 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:19,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:19,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:19,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:19,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41712 deadline: 1732743379373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:19,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:19,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:19,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41682 deadline: 1732743379374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:19,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:19,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:19,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:19,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:19,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:19,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-27T21:35:19,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:19,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. as already flushing 2024-11-27T21:35:19,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:19,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:19,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:19,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:19,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/8afc08bd82764c649ff582976f074f73 2024-11-27T21:35:19,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/2b9cfbbcd1484e91a7bb34f1d1e8db77 is 50, key is test_row_0/C:col10/1732743318213/Put/seqid=0 2024-11-27T21:35:19,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742062_1238 (size=12301) 2024-11-27T21:35:19,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-27T21:35:19,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:19,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. as already flushing 2024-11-27T21:35:19,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:19,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:19,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:19,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:19,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-27T21:35:19,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:19,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. as already flushing 2024-11-27T21:35:19,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:19,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:19,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:19,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:19,867 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/2b9cfbbcd1484e91a7bb34f1d1e8db77 2024-11-27T21:35:19,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/ebf6796361a44af38f70e11f982dda69 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/ebf6796361a44af38f70e11f982dda69 2024-11-27T21:35:19,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/ebf6796361a44af38f70e11f982dda69, entries=250, sequenceid=357, filesize=47.5 K 2024-11-27T21:35:19,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/8afc08bd82764c649ff582976f074f73 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/8afc08bd82764c649ff582976f074f73 2024-11-27T21:35:19,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/8afc08bd82764c649ff582976f074f73, entries=150, sequenceid=357, filesize=12.0 K 2024-11-27T21:35:19,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/2b9cfbbcd1484e91a7bb34f1d1e8db77 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/2b9cfbbcd1484e91a7bb34f1d1e8db77 2024-11-27T21:35:19,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/2b9cfbbcd1484e91a7bb34f1d1e8db77, entries=150, sequenceid=357, filesize=12.0 K 2024-11-27T21:35:19,887 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 4c65e64f21dac3e5db22a4c47b796905 in 1673ms, sequenceid=357, compaction requested=true 2024-11-27T21:35:19,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c65e64f21dac3e5db22a4c47b796905: 2024-11-27T21:35:19,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c65e64f21dac3e5db22a4c47b796905:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:35:19,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:19,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c65e64f21dac3e5db22a4c47b796905:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:35:19,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:19,887 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-27T21:35:19,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c65e64f21dac3e5db22a4c47b796905:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:35:19,887 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-27T21:35:19,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:35:19,891 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 174307 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-27T21:35:19,891 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62153 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-27T21:35:19,891 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): 4c65e64f21dac3e5db22a4c47b796905/A is initiating minor compaction (all files) 2024-11-27T21:35:19,891 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): 4c65e64f21dac3e5db22a4c47b796905/B is initiating minor compaction (all files) 2024-11-27T21:35:19,891 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c65e64f21dac3e5db22a4c47b796905/A in TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:19,891 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c65e64f21dac3e5db22a4c47b796905/B in TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:19,891 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/5375099c97c84ba2843abffc237e7bf4, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/d4c514cd977649f3828d2d11cc1c83c3, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/e126b4cb12b7446d83b753ce7a0998a7, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/ae1e0f6aedb04927a2e17a3f7e64d5cf, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/ebf6796361a44af38f70e11f982dda69] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp, totalSize=170.2 K 2024-11-27T21:35:19,891 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/a2d323107c52477e96543f9cfde59e4a, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/28c854374d624fc2b0b67ea7e5db2bc2, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/8eb57d46a85b4e5d95f9ef6a11f41338, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/8fd69e615af54ccba80645077664f82f, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/8afc08bd82764c649ff582976f074f73] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp, totalSize=60.7 K 2024-11-27T21:35:19,891 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:19,891 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. files: [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/5375099c97c84ba2843abffc237e7bf4, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/d4c514cd977649f3828d2d11cc1c83c3, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/e126b4cb12b7446d83b753ce7a0998a7, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/ae1e0f6aedb04927a2e17a3f7e64d5cf, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/ebf6796361a44af38f70e11f982dda69] 2024-11-27T21:35:19,891 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting a2d323107c52477e96543f9cfde59e4a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732743311062 2024-11-27T21:35:19,892 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5375099c97c84ba2843abffc237e7bf4, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732743311062 2024-11-27T21:35:19,892 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 28c854374d624fc2b0b67ea7e5db2bc2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732743311194 2024-11-27T21:35:19,892 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4c514cd977649f3828d2d11cc1c83c3, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732743311194 2024-11-27T21:35:19,892 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 8eb57d46a85b4e5d95f9ef6a11f41338, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732743311869 2024-11-27T21:35:19,892 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting e126b4cb12b7446d83b753ce7a0998a7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732743311869 2024-11-27T21:35:19,892 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fd69e615af54ccba80645077664f82f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732743314030 2024-11-27T21:35:19,893 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae1e0f6aedb04927a2e17a3f7e64d5cf, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732743314030 2024-11-27T21:35:19,893 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 8afc08bd82764c649ff582976f074f73, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732743316106 2024-11-27T21:35:19,893 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebf6796361a44af38f70e11f982dda69, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732743316078 2024-11-27T21:35:19,902 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4c65e64f21dac3e5db22a4c47b796905] 2024-11-27T21:35:19,903 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c65e64f21dac3e5db22a4c47b796905#B#compaction#199 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:19,903 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/dd6ef3f63d1049f0b9f34cad9db2190c is 50, key is test_row_0/B:col10/1732743318213/Put/seqid=0 2024-11-27T21:35:19,904 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411271ac485e43cf148f8b23a15eae8eca4a7_4c65e64f21dac3e5db22a4c47b796905 store=[table=TestAcidGuarantees family=A region=4c65e64f21dac3e5db22a4c47b796905] 2024-11-27T21:35:19,907 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411271ac485e43cf148f8b23a15eae8eca4a7_4c65e64f21dac3e5db22a4c47b796905, store=[table=TestAcidGuarantees family=A region=4c65e64f21dac3e5db22a4c47b796905] 2024-11-27T21:35:19,907 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411271ac485e43cf148f8b23a15eae8eca4a7_4c65e64f21dac3e5db22a4c47b796905 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4c65e64f21dac3e5db22a4c47b796905] 2024-11-27T21:35:19,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-27T21:35:19,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:19,908 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 4c65e64f21dac3e5db22a4c47b796905 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-27T21:35:19,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742064_1240 (size=4469) 2024-11-27T21:35:19,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127e56ce959b9c846ae8906961d30b7ca41_4c65e64f21dac3e5db22a4c47b796905 is 50, key is test_row_0/A:col10/1732743318226/Put/seqid=0 2024-11-27T21:35:19,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742063_1239 (size=13119) 2024-11-27T21:35:19,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742065_1241 (size=12454) 2024-11-27T21:35:19,927 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/dd6ef3f63d1049f0b9f34cad9db2190c as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/dd6ef3f63d1049f0b9f34cad9db2190c 2024-11-27T21:35:19,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:35:19,934 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127e56ce959b9c846ae8906961d30b7ca41_4c65e64f21dac3e5db22a4c47b796905 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127e56ce959b9c846ae8906961d30b7ca41_4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:19,935 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 4c65e64f21dac3e5db22a4c47b796905/B of 4c65e64f21dac3e5db22a4c47b796905 into dd6ef3f63d1049f0b9f34cad9db2190c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:19,935 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c65e64f21dac3e5db22a4c47b796905: 2024-11-27T21:35:19,935 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., storeName=4c65e64f21dac3e5db22a4c47b796905/B, priority=11, startTime=1732743319887; duration=0sec 2024-11-27T21:35:19,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/db8c3e9d0a56490a8cc1e84441dc3443, store: [table=TestAcidGuarantees family=A region=4c65e64f21dac3e5db22a4c47b796905] 2024-11-27T21:35:19,935 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:35:19,935 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c65e64f21dac3e5db22a4c47b796905:B 2024-11-27T21:35:19,935 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-27T21:35:19,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/db8c3e9d0a56490a8cc1e84441dc3443 is 175, key is test_row_0/A:col10/1732743318226/Put/seqid=0 2024-11-27T21:35:19,938 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62119 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-27T21:35:19,938 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): 4c65e64f21dac3e5db22a4c47b796905/C is initiating minor compaction (all files) 2024-11-27T21:35:19,938 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c65e64f21dac3e5db22a4c47b796905/C in TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:19,938 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/74dfc12f76c94f08be9c1890cc5268aa, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/3e6ae27a754244f185d79f3d9fcece65, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/af4c168809d843c0bf938187dde57acd, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/dfca400feb97498da55ae585fa5e5421, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/2b9cfbbcd1484e91a7bb34f1d1e8db77] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp, totalSize=60.7 K 2024-11-27T21:35:19,938 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 74dfc12f76c94f08be9c1890cc5268aa, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732743311062 2024-11-27T21:35:19,939 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e6ae27a754244f185d79f3d9fcece65, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732743311194 2024-11-27T21:35:19,939 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting af4c168809d843c0bf938187dde57acd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732743311869 2024-11-27T21:35:19,939 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting dfca400feb97498da55ae585fa5e5421, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732743314030 2024-11-27T21:35:19,940 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b9cfbbcd1484e91a7bb34f1d1e8db77, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732743316106 2024-11-27T21:35:19,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742066_1242 (size=31255) 2024-11-27T21:35:19,955 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c65e64f21dac3e5db22a4c47b796905#C#compaction#202 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:19,956 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/4be39788ccbe4325a1d35c0a0c0738f7 is 50, key is test_row_0/C:col10/1732743318213/Put/seqid=0 2024-11-27T21:35:19,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742067_1243 (size=13085) 2024-11-27T21:35:19,977 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/4be39788ccbe4325a1d35c0a0c0738f7 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/4be39788ccbe4325a1d35c0a0c0738f7 2024-11-27T21:35:19,982 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 4c65e64f21dac3e5db22a4c47b796905/C of 4c65e64f21dac3e5db22a4c47b796905 into 4be39788ccbe4325a1d35c0a0c0738f7(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:19,982 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c65e64f21dac3e5db22a4c47b796905: 2024-11-27T21:35:19,982 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., storeName=4c65e64f21dac3e5db22a4c47b796905/C, priority=11, startTime=1732743319887; duration=0sec 2024-11-27T21:35:19,982 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:19,982 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c65e64f21dac3e5db22a4c47b796905:C 2024-11-27T21:35:20,101 INFO [Thread-716 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-716. 2024-11-27T21:35:20,101 DEBUG [Thread-716 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:35:20,101 DEBUG [Thread-716 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:20,101 DEBUG [Thread-716 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:20,101 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:35:20,102 INFO [Thread-720 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-720. 2024-11-27T21:35:20,102 DEBUG [Thread-720 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:35:20,102 DEBUG [Thread-720 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:20,102 DEBUG [Thread-720 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:20,102 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:35:20,103 INFO [Thread-714 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-714. 2024-11-27T21:35:20,103 DEBUG [Thread-714 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:35:20,103 DEBUG [Thread-714 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:20,103 DEBUG [Thread-714 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:20,103 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:35:20,104 INFO [Thread-718 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-718. 2024-11-27T21:35:20,104 DEBUG [Thread-718 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:35:20,105 DEBUG [Thread-718 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:20,105 DEBUG [Thread-718 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:20,105 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:35:20,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-27T21:35:20,285 INFO [Thread-707 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-707. 2024-11-27T21:35:20,285 DEBUG [Thread-707 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:35:20,285 DEBUG [Thread-707 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:20,285 DEBUG [Thread-707 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:20,285 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:35:20,294 INFO [Thread-705 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-705. 2024-11-27T21:35:20,294 DEBUG [Thread-705 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:35:20,294 DEBUG [Thread-705 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:20,294 DEBUG [Thread-705 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:20,294 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:35:20,316 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c65e64f21dac3e5db22a4c47b796905#A#compaction#200 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:20,317 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/2ce09ce640e34efa929a84f19f042742 is 175, key is test_row_0/A:col10/1732743318213/Put/seqid=0 2024-11-27T21:35:20,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742068_1244 (size=32073) 2024-11-27T21:35:20,343 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=365, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/db8c3e9d0a56490a8cc1e84441dc3443 2024-11-27T21:35:20,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/4e5357995e554de097853d3d13d4daed is 50, key is test_row_0/B:col10/1732743318226/Put/seqid=0 2024-11-27T21:35:20,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742069_1245 (size=12301) 2024-11-27T21:35:20,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on 4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:20,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. as already flushing 2024-11-27T21:35:20,385 INFO [Thread-703 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-703. 2024-11-27T21:35:20,385 DEBUG [Thread-703 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:35:20,385 DEBUG [Thread-703 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:20,385 DEBUG [Thread-703 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:20,385 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:35:20,394 INFO [Thread-711 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-711. 2024-11-27T21:35:20,394 DEBUG [Thread-711 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:35:20,394 DEBUG [Thread-711 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:20,394 DEBUG [Thread-711 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:20,394 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:35:20,726 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/2ce09ce640e34efa929a84f19f042742 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/2ce09ce640e34efa929a84f19f042742 2024-11-27T21:35:20,730 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 4c65e64f21dac3e5db22a4c47b796905/A of 4c65e64f21dac3e5db22a4c47b796905 into 2ce09ce640e34efa929a84f19f042742(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:20,730 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c65e64f21dac3e5db22a4c47b796905: 2024-11-27T21:35:20,730 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., storeName=4c65e64f21dac3e5db22a4c47b796905/A, priority=11, startTime=1732743319887; duration=0sec 2024-11-27T21:35:20,730 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:20,731 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c65e64f21dac3e5db22a4c47b796905:A 2024-11-27T21:35:20,753 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/4e5357995e554de097853d3d13d4daed 2024-11-27T21:35:20,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/d95e911b7e634f869284712458c3197e is 50, key is test_row_0/C:col10/1732743318226/Put/seqid=0 2024-11-27T21:35:20,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742070_1246 (size=12301) 2024-11-27T21:35:21,164 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/d95e911b7e634f869284712458c3197e 2024-11-27T21:35:21,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/db8c3e9d0a56490a8cc1e84441dc3443 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/db8c3e9d0a56490a8cc1e84441dc3443 2024-11-27T21:35:21,173 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/db8c3e9d0a56490a8cc1e84441dc3443, entries=150, sequenceid=365, filesize=30.5 K 2024-11-27T21:35:21,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/4e5357995e554de097853d3d13d4daed as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/4e5357995e554de097853d3d13d4daed 2024-11-27T21:35:21,177 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/4e5357995e554de097853d3d13d4daed, entries=150, sequenceid=365, filesize=12.0 K 2024-11-27T21:35:21,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/d95e911b7e634f869284712458c3197e as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/d95e911b7e634f869284712458c3197e 2024-11-27T21:35:21,181 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/d95e911b7e634f869284712458c3197e, entries=150, sequenceid=365, filesize=12.0 K 2024-11-27T21:35:21,182 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=26.84 KB/27480 for 4c65e64f21dac3e5db22a4c47b796905 in 1274ms, sequenceid=365, compaction requested=false 2024-11-27T21:35:21,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 4c65e64f21dac3e5db22a4c47b796905: 2024-11-27T21:35:21,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:21,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-27T21:35:21,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-27T21:35:21,185 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-27T21:35:21,185 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1110 sec 2024-11-27T21:35:21,186 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 3.1310 sec 2024-11-27T21:35:22,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-27T21:35:22,224 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:35:22,274 INFO [Thread-709 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-709. 2024-11-27T21:35:22,274 DEBUG [Thread-709 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:35:22,274 DEBUG [Thread-709 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:22,275 DEBUG [Thread-709 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:22,275 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:35:22,275 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-27T21:35:22,275 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-27T21:35:22,275 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-27T21:35:22,275 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-27T21:35:22,275 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-27T21:35:22,275 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-27T21:35:22,275 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-27T21:35:22,275 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3547 2024-11-27T21:35:22,275 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3671 2024-11-27T21:35:22,275 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-27T21:35:22,275 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1568 2024-11-27T21:35:22,275 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4691 rows 2024-11-27T21:35:22,275 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1568 2024-11-27T21:35:22,275 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4693 rows 2024-11-27T21:35:22,275 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T21:35:22,275 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T21:35:22,275 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:22,275 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:22,276 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:35:22,276 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-27T21:35:22,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-27T21:35:22,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-27T21:35:22,279 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732743322278"}]},"ts":"1732743322278"} 2024-11-27T21:35:22,280 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-27T21:35:22,280 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-27T21:35:22,281 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T21:35:22,282 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c65e64f21dac3e5db22a4c47b796905, UNASSIGN}] 2024-11-27T21:35:22,282 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c65e64f21dac3e5db22a4c47b796905, UNASSIGN 2024-11-27T21:35:22,283 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=4c65e64f21dac3e5db22a4c47b796905, regionState=CLOSING, regionLocation=4585c39bed03,39097,1732743267048 2024-11-27T21:35:22,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c65e64f21dac3e5db22a4c47b796905, UNASSIGN because future has completed 2024-11-27T21:35:22,284 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T21:35:22,284 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048}] 2024-11-27T21:35:22,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-27T21:35:22,436 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(122): Close 4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:22,436 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T21:35:22,436 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1722): Closing 4c65e64f21dac3e5db22a4c47b796905, disabling compactions & flushes 2024-11-27T21:35:22,436 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:22,436 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:22,436 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. after waiting 0 ms 2024-11-27T21:35:22,436 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:22,437 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(2902): Flushing 4c65e64f21dac3e5db22a4c47b796905 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-27T21:35:22,442 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127589b5a6d4cd541e7a8f1d14315eead77_4c65e64f21dac3e5db22a4c47b796905 is 50, key is test_row_0/A:col10/1732743322273/Put/seqid=0 2024-11-27T21:35:22,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742071_1247 (size=12454) 2024-11-27T21:35:22,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-27T21:35:22,847 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:35:22,850 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127589b5a6d4cd541e7a8f1d14315eead77_4c65e64f21dac3e5db22a4c47b796905 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127589b5a6d4cd541e7a8f1d14315eead77_4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:22,851 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/fe6301ba1d924488888debf4c6729dd6, store: [table=TestAcidGuarantees family=A region=4c65e64f21dac3e5db22a4c47b796905] 2024-11-27T21:35:22,852 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/fe6301ba1d924488888debf4c6729dd6 is 175, key is test_row_0/A:col10/1732743322273/Put/seqid=0 2024-11-27T21:35:22,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742072_1248 (size=31255) 2024-11-27T21:35:22,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-27T21:35:23,256 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=376, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/fe6301ba1d924488888debf4c6729dd6 2024-11-27T21:35:23,263 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/0738a27778ef4f58a5dc40553e1f42a3 is 50, key is test_row_0/B:col10/1732743322273/Put/seqid=0 2024-11-27T21:35:23,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742073_1249 (size=12301) 2024-11-27T21:35:23,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-27T21:35:23,667 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/0738a27778ef4f58a5dc40553e1f42a3 2024-11-27T21:35:23,674 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/8a591f0fd441441cb820b3c26296d94a is 50, key is test_row_0/C:col10/1732743322273/Put/seqid=0 2024-11-27T21:35:23,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742074_1250 (size=12301) 2024-11-27T21:35:24,078 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/8a591f0fd441441cb820b3c26296d94a 2024-11-27T21:35:24,082 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/A/fe6301ba1d924488888debf4c6729dd6 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/fe6301ba1d924488888debf4c6729dd6 2024-11-27T21:35:24,086 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/fe6301ba1d924488888debf4c6729dd6, entries=150, sequenceid=376, filesize=30.5 K 2024-11-27T21:35:24,087 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/B/0738a27778ef4f58a5dc40553e1f42a3 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/0738a27778ef4f58a5dc40553e1f42a3 2024-11-27T21:35:24,090 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/0738a27778ef4f58a5dc40553e1f42a3, entries=150, sequenceid=376, filesize=12.0 K 2024-11-27T21:35:24,091 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/.tmp/C/8a591f0fd441441cb820b3c26296d94a as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/8a591f0fd441441cb820b3c26296d94a 2024-11-27T21:35:24,094 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/8a591f0fd441441cb820b3c26296d94a, entries=150, sequenceid=376, filesize=12.0 K 2024-11-27T21:35:24,094 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 4c65e64f21dac3e5db22a4c47b796905 in 1657ms, sequenceid=376, compaction requested=true 2024-11-27T21:35:24,095 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/4ecf5c5ffdbe49d59968dcd0667d1087, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/08bddcfae5924aee9ded0e44a7e5dc84, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/4f7b4f1da64f46c684cd5a3bdc1cfc32, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/7f84c81596a34a5a9e27abde5e185b87, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/d243c1e7b09f4ffc8794fa2995a29b4a, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/75b09b937f5644378763bde581cff786, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/903406b63d414a5da581436e2f6556af, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/3064ddc776d4428a9a1d92895ba302be, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/3fbda9112e044405be549031773b5024, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/a3945c7d73714ab7996cc0b2608c7a99, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/e2767feb957e4a699ed74830bb5d16cf, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/ada7412ddd6144ca8ef648bfa13aa80e, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/4a8787c5c2614022bd929509aa03dbe7, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/0c101e74219d4853a49ba63cf3b47a5d, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/2da9b094e43b4e518e09672e7236020c, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/6a2107cabc644145a21aa53ccb043e24, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/2ea03a414f094f2395de3dc71625b7cb, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/e3665866464840429d41e096d700d0cb, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/a04e3d5113d34138b09f6ae855f7857b, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/5375099c97c84ba2843abffc237e7bf4, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/d4c514cd977649f3828d2d11cc1c83c3, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/e126b4cb12b7446d83b753ce7a0998a7, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/ae1e0f6aedb04927a2e17a3f7e64d5cf, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/ebf6796361a44af38f70e11f982dda69] to archive 2024-11-27T21:35:24,096 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T21:35:24,097 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/4ecf5c5ffdbe49d59968dcd0667d1087 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/4ecf5c5ffdbe49d59968dcd0667d1087 2024-11-27T21:35:24,098 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/08bddcfae5924aee9ded0e44a7e5dc84 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/08bddcfae5924aee9ded0e44a7e5dc84 2024-11-27T21:35:24,099 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/4f7b4f1da64f46c684cd5a3bdc1cfc32 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/4f7b4f1da64f46c684cd5a3bdc1cfc32 2024-11-27T21:35:24,100 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/7f84c81596a34a5a9e27abde5e185b87 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/7f84c81596a34a5a9e27abde5e185b87 2024-11-27T21:35:24,101 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/d243c1e7b09f4ffc8794fa2995a29b4a to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/d243c1e7b09f4ffc8794fa2995a29b4a 2024-11-27T21:35:24,102 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/75b09b937f5644378763bde581cff786 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/75b09b937f5644378763bde581cff786 2024-11-27T21:35:24,103 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/903406b63d414a5da581436e2f6556af to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/903406b63d414a5da581436e2f6556af 2024-11-27T21:35:24,104 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/3064ddc776d4428a9a1d92895ba302be to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/3064ddc776d4428a9a1d92895ba302be 2024-11-27T21:35:24,105 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/3fbda9112e044405be549031773b5024 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/3fbda9112e044405be549031773b5024 2024-11-27T21:35:24,106 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/a3945c7d73714ab7996cc0b2608c7a99 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/a3945c7d73714ab7996cc0b2608c7a99 2024-11-27T21:35:24,107 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/e2767feb957e4a699ed74830bb5d16cf to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/e2767feb957e4a699ed74830bb5d16cf 2024-11-27T21:35:24,108 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/ada7412ddd6144ca8ef648bfa13aa80e to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/ada7412ddd6144ca8ef648bfa13aa80e 2024-11-27T21:35:24,109 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/4a8787c5c2614022bd929509aa03dbe7 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/4a8787c5c2614022bd929509aa03dbe7 2024-11-27T21:35:24,110 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/0c101e74219d4853a49ba63cf3b47a5d to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/0c101e74219d4853a49ba63cf3b47a5d 2024-11-27T21:35:24,111 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/2da9b094e43b4e518e09672e7236020c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/2da9b094e43b4e518e09672e7236020c 2024-11-27T21:35:24,112 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/6a2107cabc644145a21aa53ccb043e24 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/6a2107cabc644145a21aa53ccb043e24 2024-11-27T21:35:24,113 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/2ea03a414f094f2395de3dc71625b7cb to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/2ea03a414f094f2395de3dc71625b7cb 2024-11-27T21:35:24,114 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/e3665866464840429d41e096d700d0cb to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/e3665866464840429d41e096d700d0cb 2024-11-27T21:35:24,115 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/a04e3d5113d34138b09f6ae855f7857b to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/a04e3d5113d34138b09f6ae855f7857b 2024-11-27T21:35:24,116 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/5375099c97c84ba2843abffc237e7bf4 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/5375099c97c84ba2843abffc237e7bf4 2024-11-27T21:35:24,117 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/d4c514cd977649f3828d2d11cc1c83c3 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/d4c514cd977649f3828d2d11cc1c83c3 2024-11-27T21:35:24,118 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/e126b4cb12b7446d83b753ce7a0998a7 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/e126b4cb12b7446d83b753ce7a0998a7 2024-11-27T21:35:24,119 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/ae1e0f6aedb04927a2e17a3f7e64d5cf to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/ae1e0f6aedb04927a2e17a3f7e64d5cf 2024-11-27T21:35:24,120 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/ebf6796361a44af38f70e11f982dda69 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/ebf6796361a44af38f70e11f982dda69 2024-11-27T21:35:24,121 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/6d0e9bedae814a02a672fec49697df7c, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/4521380bcc3e4f99ae4122505561aaa8, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/8393b83ac73248ce93b7ce82013bccf2, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/9f3b068438e94d1ab4baab4c7d3e523b, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/e3996131b4dd4340b44b5655754a152d, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/16da5bcc63164282842de3fbabe4df31, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/cc501cf87b4242939a3b8f5aec8b9e20, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/67b5f27989b84b5e91d71e70a291cd24, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/688e026109224ae6beefeea5a5a74107, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/56ce1f6e5a50484cb54dfb3dbdcceea4, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/5ed2fdd940fa4ad0a47a8ce8d8f026a9, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/bbfb5343fbe1473b97537d9f8dd81570, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/eac5727c995542eea7b66a925ae703e3, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/18744fd273eb41f8b4b8061afba6bf44, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/2b05cc9f0552404ca7ef7cd0c022bee2, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/3e8c5cf4edd84752b8ca33d56836d8a0, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/f1a68666f8c840749a618c3955345616, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/4f617b5d64c34721a31c5bbefd53ca99, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/a2d323107c52477e96543f9cfde59e4a, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/df03c05d59e246b6ad4274d7c8b2cd41, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/28c854374d624fc2b0b67ea7e5db2bc2, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/8eb57d46a85b4e5d95f9ef6a11f41338, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/8fd69e615af54ccba80645077664f82f, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/8afc08bd82764c649ff582976f074f73] to archive 2024-11-27T21:35:24,122 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T21:35:24,123 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/6d0e9bedae814a02a672fec49697df7c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/6d0e9bedae814a02a672fec49697df7c 2024-11-27T21:35:24,125 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/4521380bcc3e4f99ae4122505561aaa8 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/4521380bcc3e4f99ae4122505561aaa8 2024-11-27T21:35:24,125 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/8393b83ac73248ce93b7ce82013bccf2 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/8393b83ac73248ce93b7ce82013bccf2 2024-11-27T21:35:24,126 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/9f3b068438e94d1ab4baab4c7d3e523b to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/9f3b068438e94d1ab4baab4c7d3e523b 2024-11-27T21:35:24,127 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/e3996131b4dd4340b44b5655754a152d to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/e3996131b4dd4340b44b5655754a152d 2024-11-27T21:35:24,128 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/16da5bcc63164282842de3fbabe4df31 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/16da5bcc63164282842de3fbabe4df31 2024-11-27T21:35:24,129 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/cc501cf87b4242939a3b8f5aec8b9e20 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/cc501cf87b4242939a3b8f5aec8b9e20 2024-11-27T21:35:24,130 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/67b5f27989b84b5e91d71e70a291cd24 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/67b5f27989b84b5e91d71e70a291cd24 2024-11-27T21:35:24,131 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/688e026109224ae6beefeea5a5a74107 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/688e026109224ae6beefeea5a5a74107 2024-11-27T21:35:24,132 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/56ce1f6e5a50484cb54dfb3dbdcceea4 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/56ce1f6e5a50484cb54dfb3dbdcceea4 2024-11-27T21:35:24,133 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/5ed2fdd940fa4ad0a47a8ce8d8f026a9 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/5ed2fdd940fa4ad0a47a8ce8d8f026a9 2024-11-27T21:35:24,134 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/bbfb5343fbe1473b97537d9f8dd81570 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/bbfb5343fbe1473b97537d9f8dd81570 2024-11-27T21:35:24,135 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/eac5727c995542eea7b66a925ae703e3 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/eac5727c995542eea7b66a925ae703e3 2024-11-27T21:35:24,136 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/18744fd273eb41f8b4b8061afba6bf44 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/18744fd273eb41f8b4b8061afba6bf44 2024-11-27T21:35:24,137 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/2b05cc9f0552404ca7ef7cd0c022bee2 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/2b05cc9f0552404ca7ef7cd0c022bee2 2024-11-27T21:35:24,138 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/3e8c5cf4edd84752b8ca33d56836d8a0 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/3e8c5cf4edd84752b8ca33d56836d8a0 2024-11-27T21:35:24,139 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/f1a68666f8c840749a618c3955345616 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/f1a68666f8c840749a618c3955345616 2024-11-27T21:35:24,140 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/4f617b5d64c34721a31c5bbefd53ca99 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/4f617b5d64c34721a31c5bbefd53ca99 2024-11-27T21:35:24,141 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/a2d323107c52477e96543f9cfde59e4a to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/a2d323107c52477e96543f9cfde59e4a 2024-11-27T21:35:24,142 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/df03c05d59e246b6ad4274d7c8b2cd41 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/df03c05d59e246b6ad4274d7c8b2cd41 2024-11-27T21:35:24,143 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/28c854374d624fc2b0b67ea7e5db2bc2 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/28c854374d624fc2b0b67ea7e5db2bc2 2024-11-27T21:35:24,144 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/8eb57d46a85b4e5d95f9ef6a11f41338 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/8eb57d46a85b4e5d95f9ef6a11f41338 2024-11-27T21:35:24,145 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/8fd69e615af54ccba80645077664f82f to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/8fd69e615af54ccba80645077664f82f 2024-11-27T21:35:24,146 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/8afc08bd82764c649ff582976f074f73 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/8afc08bd82764c649ff582976f074f73 2024-11-27T21:35:24,147 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/e8f1a0f4882e4ffba03bb678822007df, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/9ccb0ef2f24a46028ac556466bfe691e, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/227ab7724ffe40eb8a90cb7a808ba263, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/03ad5007dd714131be9735a4f0c8bb7d, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/3a23bf4d7930473198a630fe6f9db4e2, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/38750da6aa85410c83275ebdc5a6786b, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/16cbba6d7e6b4a4dbce2ac5ff4aa371d, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/3830173ad7874577b1265f4c4cc578ee, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/67bfa5beaa84419db37707b9bb717d2f, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/ff492250e8274e4193d935c5f4451009, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/13a67ca3871c434ab6eb034636ed9b3e, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/849bb24a746146e1883b3e335fbd7dc6, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/ecb58b15d0714b26827d10f3397a1a56, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/1639feeceda445ce88a113b28b7942f8, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/07415fc793644c33b863bc326611a357, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/b8ddec8c331b4eeda04b4dde3a53cdae, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/cef0ee1969c84b34884f2a000b0d1278, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/74dfc12f76c94f08be9c1890cc5268aa, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/07399186f1814de882ff28ce63a2ea53, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/3e6ae27a754244f185d79f3d9fcece65, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/af4c168809d843c0bf938187dde57acd, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/dfca400feb97498da55ae585fa5e5421, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/2b9cfbbcd1484e91a7bb34f1d1e8db77] to archive 2024-11-27T21:35:24,148 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T21:35:24,149 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/e8f1a0f4882e4ffba03bb678822007df to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/e8f1a0f4882e4ffba03bb678822007df 2024-11-27T21:35:24,150 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/9ccb0ef2f24a46028ac556466bfe691e to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/9ccb0ef2f24a46028ac556466bfe691e 2024-11-27T21:35:24,151 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/227ab7724ffe40eb8a90cb7a808ba263 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/227ab7724ffe40eb8a90cb7a808ba263 2024-11-27T21:35:24,152 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/03ad5007dd714131be9735a4f0c8bb7d to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/03ad5007dd714131be9735a4f0c8bb7d 2024-11-27T21:35:24,153 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/3a23bf4d7930473198a630fe6f9db4e2 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/3a23bf4d7930473198a630fe6f9db4e2 2024-11-27T21:35:24,154 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/38750da6aa85410c83275ebdc5a6786b to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/38750da6aa85410c83275ebdc5a6786b 2024-11-27T21:35:24,155 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/16cbba6d7e6b4a4dbce2ac5ff4aa371d to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/16cbba6d7e6b4a4dbce2ac5ff4aa371d 2024-11-27T21:35:24,156 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/3830173ad7874577b1265f4c4cc578ee to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/3830173ad7874577b1265f4c4cc578ee 2024-11-27T21:35:24,157 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/67bfa5beaa84419db37707b9bb717d2f to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/67bfa5beaa84419db37707b9bb717d2f 2024-11-27T21:35:24,158 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/ff492250e8274e4193d935c5f4451009 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/ff492250e8274e4193d935c5f4451009 2024-11-27T21:35:24,159 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/13a67ca3871c434ab6eb034636ed9b3e to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/13a67ca3871c434ab6eb034636ed9b3e 2024-11-27T21:35:24,160 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/849bb24a746146e1883b3e335fbd7dc6 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/849bb24a746146e1883b3e335fbd7dc6 2024-11-27T21:35:24,161 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/ecb58b15d0714b26827d10f3397a1a56 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/ecb58b15d0714b26827d10f3397a1a56 2024-11-27T21:35:24,162 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/1639feeceda445ce88a113b28b7942f8 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/1639feeceda445ce88a113b28b7942f8 2024-11-27T21:35:24,163 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/07415fc793644c33b863bc326611a357 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/07415fc793644c33b863bc326611a357 2024-11-27T21:35:24,164 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/b8ddec8c331b4eeda04b4dde3a53cdae to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/b8ddec8c331b4eeda04b4dde3a53cdae 2024-11-27T21:35:24,165 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/cef0ee1969c84b34884f2a000b0d1278 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/cef0ee1969c84b34884f2a000b0d1278 2024-11-27T21:35:24,166 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/74dfc12f76c94f08be9c1890cc5268aa to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/74dfc12f76c94f08be9c1890cc5268aa 2024-11-27T21:35:24,167 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/07399186f1814de882ff28ce63a2ea53 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/07399186f1814de882ff28ce63a2ea53 2024-11-27T21:35:24,168 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/3e6ae27a754244f185d79f3d9fcece65 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/3e6ae27a754244f185d79f3d9fcece65 2024-11-27T21:35:24,169 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/af4c168809d843c0bf938187dde57acd to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/af4c168809d843c0bf938187dde57acd 2024-11-27T21:35:24,170 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/dfca400feb97498da55ae585fa5e5421 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/dfca400feb97498da55ae585fa5e5421 2024-11-27T21:35:24,171 DEBUG [StoreCloser-TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/2b9cfbbcd1484e91a7bb34f1d1e8db77 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/2b9cfbbcd1484e91a7bb34f1d1e8db77 2024-11-27T21:35:24,175 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/recovered.edits/379.seqid, newMaxSeqId=379, maxSeqId=4 2024-11-27T21:35:24,176 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. 2024-11-27T21:35:24,176 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1676): Region close journal for 4c65e64f21dac3e5db22a4c47b796905: Waiting for close lock at 1732743322436Running coprocessor pre-close hooks at 1732743322436Disabling compacts and flushes for region at 1732743322436Disabling writes for close at 1732743322436Obtaining lock to block concurrent updates at 1732743322437 (+1 ms)Preparing flush snapshotting stores in 4c65e64f21dac3e5db22a4c47b796905 at 1732743322437Finished memstore snapshotting TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732743322437Flushing stores of TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905. at 1732743322437Flushing 4c65e64f21dac3e5db22a4c47b796905/A: creating writer at 1732743322437Flushing mob file 4c65e64f21dac3e5db22a4c47b796905/A: appending metadata at 1732743322442 (+5 ms)Flushing mob file 4c65e64f21dac3e5db22a4c47b796905/A: closing flushed file at 1732743322442Flushing 4c65e64f21dac3e5db22a4c47b796905/A: appending metadata at 1732743322851 (+409 ms)Flushing 4c65e64f21dac3e5db22a4c47b796905/A: closing flushed file at 1732743322851Flushing 4c65e64f21dac3e5db22a4c47b796905/B: creating writer at 1732743323260 (+409 ms)Flushing 4c65e64f21dac3e5db22a4c47b796905/B: appending metadata at 1732743323262 (+2 ms)Flushing 4c65e64f21dac3e5db22a4c47b796905/B: closing flushed file at 1732743323262Flushing 4c65e64f21dac3e5db22a4c47b796905/C: creating writer at 1732743323671 (+409 ms)Flushing 4c65e64f21dac3e5db22a4c47b796905/C: appending metadata at 1732743323673 (+2 ms)Flushing 4c65e64f21dac3e5db22a4c47b796905/C: closing flushed file at 1732743323673Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a7f8e5b: reopening flushed file at 1732743324082 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63328f2e: reopening flushed file at 1732743324086 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f1b05eb: reopening flushed file at 1732743324090 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 4c65e64f21dac3e5db22a4c47b796905 in 1657ms, sequenceid=376, compaction requested=true at 1732743324094 (+4 ms)Writing region close event to WAL at 1732743324172 (+78 ms)Running coprocessor post-close hooks at 1732743324176 (+4 ms)Closed at 1732743324176 2024-11-27T21:35:24,178 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(157): Closed 4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:24,179 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=4c65e64f21dac3e5db22a4c47b796905, regionState=CLOSED 2024-11-27T21:35:24,180 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 because future has completed 2024-11-27T21:35:24,183 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-27T21:35:24,183 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; CloseRegionProcedure 4c65e64f21dac3e5db22a4c47b796905, server=4585c39bed03,39097,1732743267048 in 1.8970 sec 2024-11-27T21:35:24,184 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-27T21:35:24,184 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c65e64f21dac3e5db22a4c47b796905, UNASSIGN in 1.9010 sec 2024-11-27T21:35:24,186 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-27T21:35:24,186 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9040 sec 2024-11-27T21:35:24,187 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732743324187"}]},"ts":"1732743324187"} 2024-11-27T21:35:24,188 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-27T21:35:24,188 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-27T21:35:24,190 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9130 sec 2024-11-27T21:35:24,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-27T21:35:24,423 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:35:24,424 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-27T21:35:24,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:35:24,425 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=64, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:35:24,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-27T21:35:24,426 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=64, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:35:24,429 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:24,431 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A, FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B, FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C, FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/recovered.edits] 2024-11-27T21:35:24,433 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/2ce09ce640e34efa929a84f19f042742 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/2ce09ce640e34efa929a84f19f042742 2024-11-27T21:35:24,435 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/db8c3e9d0a56490a8cc1e84441dc3443 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/db8c3e9d0a56490a8cc1e84441dc3443 2024-11-27T21:35:24,436 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/fe6301ba1d924488888debf4c6729dd6 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/A/fe6301ba1d924488888debf4c6729dd6 2024-11-27T21:35:24,438 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/0738a27778ef4f58a5dc40553e1f42a3 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/0738a27778ef4f58a5dc40553e1f42a3 2024-11-27T21:35:24,439 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/4e5357995e554de097853d3d13d4daed to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/4e5357995e554de097853d3d13d4daed 2024-11-27T21:35:24,440 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/dd6ef3f63d1049f0b9f34cad9db2190c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/B/dd6ef3f63d1049f0b9f34cad9db2190c 2024-11-27T21:35:24,442 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/4be39788ccbe4325a1d35c0a0c0738f7 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/4be39788ccbe4325a1d35c0a0c0738f7 2024-11-27T21:35:24,443 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/8a591f0fd441441cb820b3c26296d94a to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/8a591f0fd441441cb820b3c26296d94a 2024-11-27T21:35:24,444 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/d95e911b7e634f869284712458c3197e to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/C/d95e911b7e634f869284712458c3197e 2024-11-27T21:35:24,446 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/recovered.edits/379.seqid to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905/recovered.edits/379.seqid 2024-11-27T21:35:24,447 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:24,447 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-27T21:35:24,447 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-27T21:35:24,448 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-27T21:35:24,452 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112702e9001262184961bc490af6495c4e6c_4c65e64f21dac3e5db22a4c47b796905 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112702e9001262184961bc490af6495c4e6c_4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:24,453 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411270e24740406b547b2adcb3719345252b3_4c65e64f21dac3e5db22a4c47b796905 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411270e24740406b547b2adcb3719345252b3_4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:24,454 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112711a9966a241d45b1a7f5576ffc403afe_4c65e64f21dac3e5db22a4c47b796905 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112711a9966a241d45b1a7f5576ffc403afe_4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:24,455 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112719b948bf18e74f45b462b21209368007_4c65e64f21dac3e5db22a4c47b796905 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112719b948bf18e74f45b462b21209368007_4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:24,456 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112734cd4d2aba1f438aa743b7e52fe8a0b9_4c65e64f21dac3e5db22a4c47b796905 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112734cd4d2aba1f438aa743b7e52fe8a0b9_4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:24,457 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127589b5a6d4cd541e7a8f1d14315eead77_4c65e64f21dac3e5db22a4c47b796905 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127589b5a6d4cd541e7a8f1d14315eead77_4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:24,458 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112762c9e887d0774ed3b556da4f2f0bd1e6_4c65e64f21dac3e5db22a4c47b796905 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112762c9e887d0774ed3b556da4f2f0bd1e6_4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:24,459 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411276feb782ffaa7455697ba28dd86d964af_4c65e64f21dac3e5db22a4c47b796905 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411276feb782ffaa7455697ba28dd86d964af_4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:24,460 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411277345dcbc486641dfba65b6a3506e5796_4c65e64f21dac3e5db22a4c47b796905 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411277345dcbc486641dfba65b6a3506e5796_4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:24,461 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411278d9b6909826141c5980d4396f8282689_4c65e64f21dac3e5db22a4c47b796905 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411278d9b6909826141c5980d4396f8282689_4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:24,463 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279a05153aca5447728f9b6e0a6a49413a_4c65e64f21dac3e5db22a4c47b796905 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279a05153aca5447728f9b6e0a6a49413a_4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:24,464 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a72ecb9cc0e44c8c8c8b3b0c11ca7048_4c65e64f21dac3e5db22a4c47b796905 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a72ecb9cc0e44c8c8c8b3b0c11ca7048_4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:24,465 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a97a09d1d2f24ed3bc416aeb7e2017e7_4c65e64f21dac3e5db22a4c47b796905 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a97a09d1d2f24ed3bc416aeb7e2017e7_4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:24,466 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127aa9892d6c76a48efa4aab25706f5da3d_4c65e64f21dac3e5db22a4c47b796905 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127aa9892d6c76a48efa4aab25706f5da3d_4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:24,467 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127bda8b8f7aad64077a2e3089f25ec5d18_4c65e64f21dac3e5db22a4c47b796905 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127bda8b8f7aad64077a2e3089f25ec5d18_4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:24,468 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d0bdeab3d2b84029a5c444c82065890e_4c65e64f21dac3e5db22a4c47b796905 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d0bdeab3d2b84029a5c444c82065890e_4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:24,469 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d5277ff6cfed47928d0120fcbdd39106_4c65e64f21dac3e5db22a4c47b796905 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d5277ff6cfed47928d0120fcbdd39106_4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:24,471 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d9878bae316f4964bebe15c2b7d5a7b5_4c65e64f21dac3e5db22a4c47b796905 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d9878bae316f4964bebe15c2b7d5a7b5_4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:24,472 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127e56ce959b9c846ae8906961d30b7ca41_4c65e64f21dac3e5db22a4c47b796905 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127e56ce959b9c846ae8906961d30b7ca41_4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:24,473 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f34da35d48584c52bdf4b8a7bf557531_4c65e64f21dac3e5db22a4c47b796905 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f34da35d48584c52bdf4b8a7bf557531_4c65e64f21dac3e5db22a4c47b796905 2024-11-27T21:35:24,474 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-27T21:35:24,476 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=64, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:35:24,478 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-27T21:35:24,480 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-27T21:35:24,481 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=64, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:35:24,481 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-27T21:35:24,482 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732743324481"}]},"ts":"9223372036854775807"} 2024-11-27T21:35:24,484 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-27T21:35:24,484 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 4c65e64f21dac3e5db22a4c47b796905, NAME => 'TestAcidGuarantees,,1732743296934.4c65e64f21dac3e5db22a4c47b796905.', STARTKEY => '', ENDKEY => ''}] 2024-11-27T21:35:24,484 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-27T21:35:24,484 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732743324484"}]},"ts":"9223372036854775807"} 2024-11-27T21:35:24,485 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-27T21:35:24,486 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=64, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:35:24,487 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 62 msec 2024-11-27T21:35:24,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-27T21:35:24,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-27T21:35:24,534 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:35:24,543 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=242 (was 239) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/208c7222-23c4-3618-cb4b-4763d96cc3c1/cluster_148e8983-f870-82f4-55b3-08390f1cbfef/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/208c7222-23c4-3618-cb4b-4763d96cc3c1/cluster_148e8983-f870-82f4-55b3-08390f1cbfef/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1911604878_22 at /127.0.0.1:57280 [Waiting for operation #634] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_941021170_22 at /127.0.0.1:50434 [Waiting for operation #259] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=317 (was 341), ProcessCount=11 (was 11), AvailableMemoryMB=8469 (was 8554) 2024-11-27T21:35:24,552 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=242, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=317, ProcessCount=11, AvailableMemoryMB=8468 2024-11-27T21:35:24,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T21:35:24,554 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T21:35:24,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=65, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-27T21:35:24,555 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T21:35:24,556 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:35:24,556 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 65 2024-11-27T21:35:24,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-27T21:35:24,556 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T21:35:24,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742075_1251 (size=959) 2024-11-27T21:35:24,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-27T21:35:24,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-27T21:35:24,964 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d 2024-11-27T21:35:24,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742076_1252 (size=53) 2024-11-27T21:35:25,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-27T21:35:25,212 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T21:35:25,370 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T21:35:25,371 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d9a1bb9b05a472888bd945369e4af06a, disabling compactions & flushes 2024-11-27T21:35:25,371 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:25,371 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:25,371 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. after waiting 0 ms 2024-11-27T21:35:25,371 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:25,371 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:25,371 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d9a1bb9b05a472888bd945369e4af06a: Waiting for close lock at 1732743325370Disabling compacts and flushes for region at 1732743325370Disabling writes for close at 1732743325371 (+1 ms)Writing region close event to WAL at 1732743325371Closed at 1732743325371 2024-11-27T21:35:25,372 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T21:35:25,372 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732743325372"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732743325372"}]},"ts":"1732743325372"} 2024-11-27T21:35:25,373 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T21:35:25,374 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T21:35:25,374 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732743325374"}]},"ts":"1732743325374"} 2024-11-27T21:35:25,375 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-27T21:35:25,376 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d9a1bb9b05a472888bd945369e4af06a, ASSIGN}] 2024-11-27T21:35:25,377 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d9a1bb9b05a472888bd945369e4af06a, ASSIGN 2024-11-27T21:35:25,377 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d9a1bb9b05a472888bd945369e4af06a, ASSIGN; state=OFFLINE, location=4585c39bed03,39097,1732743267048; forceNewPlan=false, retain=false 2024-11-27T21:35:25,528 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=d9a1bb9b05a472888bd945369e4af06a, regionState=OPENING, regionLocation=4585c39bed03,39097,1732743267048 2024-11-27T21:35:25,529 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d9a1bb9b05a472888bd945369e4af06a, ASSIGN because future has completed 2024-11-27T21:35:25,530 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048}] 2024-11-27T21:35:25,684 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:25,684 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7752): Opening region: {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} 2024-11-27T21:35:25,685 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d9a1bb9b05a472888bd945369e4af06a 2024-11-27T21:35:25,685 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T21:35:25,685 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7794): checking encryption for d9a1bb9b05a472888bd945369e4af06a 2024-11-27T21:35:25,685 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7797): checking classloading for d9a1bb9b05a472888bd945369e4af06a 2024-11-27T21:35:25,686 INFO [StoreOpener-d9a1bb9b05a472888bd945369e4af06a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d9a1bb9b05a472888bd945369e4af06a 2024-11-27T21:35:25,687 INFO [StoreOpener-d9a1bb9b05a472888bd945369e4af06a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9a1bb9b05a472888bd945369e4af06a columnFamilyName A 2024-11-27T21:35:25,687 DEBUG [StoreOpener-d9a1bb9b05a472888bd945369e4af06a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:35:25,688 INFO [StoreOpener-d9a1bb9b05a472888bd945369e4af06a-1 {}] regionserver.HStore(327): Store=d9a1bb9b05a472888bd945369e4af06a/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:35:25,688 INFO [StoreOpener-d9a1bb9b05a472888bd945369e4af06a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d9a1bb9b05a472888bd945369e4af06a 2024-11-27T21:35:25,689 INFO [StoreOpener-d9a1bb9b05a472888bd945369e4af06a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9a1bb9b05a472888bd945369e4af06a columnFamilyName B 2024-11-27T21:35:25,689 DEBUG [StoreOpener-d9a1bb9b05a472888bd945369e4af06a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:35:25,690 INFO [StoreOpener-d9a1bb9b05a472888bd945369e4af06a-1 {}] regionserver.HStore(327): Store=d9a1bb9b05a472888bd945369e4af06a/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:35:25,690 INFO [StoreOpener-d9a1bb9b05a472888bd945369e4af06a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d9a1bb9b05a472888bd945369e4af06a 2024-11-27T21:35:25,691 INFO [StoreOpener-d9a1bb9b05a472888bd945369e4af06a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9a1bb9b05a472888bd945369e4af06a columnFamilyName C 2024-11-27T21:35:25,691 DEBUG [StoreOpener-d9a1bb9b05a472888bd945369e4af06a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:35:25,691 INFO [StoreOpener-d9a1bb9b05a472888bd945369e4af06a-1 {}] regionserver.HStore(327): Store=d9a1bb9b05a472888bd945369e4af06a/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:35:25,691 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1038): replaying wal for d9a1bb9b05a472888bd945369e4af06a 2024-11-27T21:35:25,692 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a 2024-11-27T21:35:25,692 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a 2024-11-27T21:35:25,693 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1048): stopping wal replay for d9a1bb9b05a472888bd945369e4af06a 2024-11-27T21:35:25,693 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1060): Cleaning up temporary data for d9a1bb9b05a472888bd945369e4af06a 2024-11-27T21:35:25,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-27T21:35:25,693 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T21:35:25,694 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1093): writing seq id for d9a1bb9b05a472888bd945369e4af06a 2024-11-27T21:35:25,696 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T21:35:25,696 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1114): Opened d9a1bb9b05a472888bd945369e4af06a; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60594839, jitterRate=-0.09706653654575348}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T21:35:25,697 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d9a1bb9b05a472888bd945369e4af06a 2024-11-27T21:35:25,697 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1006): Region open journal for d9a1bb9b05a472888bd945369e4af06a: Running coprocessor pre-open hook at 1732743325685Writing region info on filesystem at 1732743325685Initializing all the Stores at 1732743325686 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732743325686Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732743325686Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732743325686Cleaning up temporary data from old regions at 1732743325693 (+7 ms)Running coprocessor post-open hooks at 1732743325697 (+4 ms)Region opened successfully at 1732743325697 2024-11-27T21:35:25,698 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., pid=67, masterSystemTime=1732743325681 2024-11-27T21:35:25,700 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:25,700 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:25,700 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=d9a1bb9b05a472888bd945369e4af06a, regionState=OPEN, openSeqNum=2, regionLocation=4585c39bed03,39097,1732743267048 2024-11-27T21:35:25,701 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 because future has completed 2024-11-27T21:35:25,703 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-27T21:35:25,703 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 in 172 msec 2024-11-27T21:35:25,705 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-11-27T21:35:25,705 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d9a1bb9b05a472888bd945369e4af06a, ASSIGN in 327 msec 2024-11-27T21:35:25,705 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T21:35:25,706 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732743325706"}]},"ts":"1732743325706"} 2024-11-27T21:35:25,707 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-27T21:35:25,707 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T21:35:25,709 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1530 sec 2024-11-27T21:35:26,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-27T21:35:26,704 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:35:26,705 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c45ccf4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:26,705 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:35:26,705 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:35:26,706 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:35:26,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:35:26,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:35:26,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cb7e29d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:26,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:35:26,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:35:26,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:26,708 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56840, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:35:26,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@322397bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:26,709 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:35:26,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:35:26,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:35:26,711 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60108, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:35:26,711 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T21:35:26,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 4585c39bed03,34433,1732743266181 2024-11-27T21:35:26,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5d078ee5 2024-11-27T21:35:26,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T21:35:26,713 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56846, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T21:35:26,714 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e140226, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:26,714 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:35:26,714 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:35:26,715 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:35:26,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:35:26,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:35:26,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cfe9b82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:26,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:35:26,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:35:26,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:26,716 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56868, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:35:26,716 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@373e7849, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:26,717 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@279bf308, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:26,717 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:35:26,717 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:35:26,718 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:35:26,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:35:26,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:35:26,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40a52fff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:26,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:35:26,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:35:26,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:26,719 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56880, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:35:26,719 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57d6fd8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:26,720 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ba0760, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:26,720 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:35:26,720 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:35:26,720 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:35:26,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:35:26,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:35:26,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@282f8ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:26,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:35:26,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:35:26,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:26,722 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56896, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:35:26,722 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4641ed61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:26,723 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@102b3c4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:26,723 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:35:26,723 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:35:26,723 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:35:26,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:35:26,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:35:26,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f6910fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:26,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:35:26,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:35:26,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:26,725 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56904, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:35:26,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@250cab36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:26,725 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@574b1ec0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:26,725 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:35:26,726 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:35:26,726 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:35:26,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:35:26,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:35:26,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a3a1315, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:26,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:35:26,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:35:26,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:26,727 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56926, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:35:26,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30e248e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:26,728 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b646580, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:26,728 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:35:26,728 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:35:26,729 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:35:26,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:35:26,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:35:26,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11fbd8a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:26,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:35:26,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:35:26,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:26,730 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56940, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:35:26,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@476f3542, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:26,731 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c85ada2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:26,731 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:35:26,731 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:35:26,731 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:35:26,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:35:26,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:35:26,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dcd2941, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:26,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:35:26,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:35:26,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:26,733 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:35:26,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@253fb8d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:26,734 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3660e239, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:26,734 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:35:26,734 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:35:26,734 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:35:26,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:35:26,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:35:26,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7855f80c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:26,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:35:26,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:35:26,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:26,735 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56988, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:35:26,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@493fe7d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:26,736 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67353b2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:26,736 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:35:26,736 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:35:26,737 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:35:26,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:35:26,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:35:26,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35439a55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:26,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:35:26,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:35:26,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:26,738 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57004, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:35:26,738 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@385b6908, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:26,739 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@364bb93f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:26,739 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:35:26,739 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:35:26,739 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:35:26,739 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:35:26,739 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:35:26,740 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@644f46e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:26,740 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:35:26,740 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:35:26,740 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:26,740 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57020, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:35:26,741 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5edd0e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:26,741 DEBUG [Thread-1166 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:35:26,741 DEBUG [Thread-1164 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:35:26,742 DEBUG [Thread-1175 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:35:26,742 DEBUG [Thread-1168 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:35:26,742 DEBUG [Thread-1183 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:35:26,742 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:35:26,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:35:26,743 DEBUG [Thread-1177 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:35:26,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:35:26,743 DEBUG [Thread-1166 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:35:26,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:35:26,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:35:26,743 DEBUG [Thread-1181 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:35:26,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-27T21:35:26,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:35:26,744 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60124, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:35:26,744 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:35:26,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:35:26,744 DEBUG [Thread-1172 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:35:26,744 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:35:26,744 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60132, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:35:26,745 DEBUG [Thread-1170 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:35:26,745 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:35:26,745 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60152, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:35:26,746 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60136, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:35:26,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:35:26,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:35:26,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2] 2024-11-27T21:35:26,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-27T21:35:26,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2] 2024-11-27T21:35:26,747 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:35:26,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2] 2024-11-27T21:35:26,747 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:35:26,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:35:26,746 DEBUG [Thread-1166 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:35:26,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:35:26,747 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60164, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:35:26,748 DEBUG [Thread-1179 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:35:26,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:35:26,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2] 2024-11-27T21:35:26,749 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60180, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:35:26,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:35:26,749 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60194, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:35:26,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2] 2024-11-27T21:35:26,749 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:35:26,750 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:35:26,750 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60200, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:35:26,751 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:35:26,751 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:35:26,751 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60210, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:35:26,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2] 2024-11-27T21:35:26,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2] 2024-11-27T21:35:26,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2] 2024-11-27T21:35:26,752 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60216, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:35:26,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2] 2024-11-27T21:35:26,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2] 2024-11-27T21:35:26,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on d9a1bb9b05a472888bd945369e4af06a 2024-11-27T21:35:26,758 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:35:26,758 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d9a1bb9b05a472888bd945369e4af06a 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-27T21:35:26,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/d7765506bb4148a38eff6b4b1260ad90 is 50, key is test_row_0/A:col10/1732743326758/Put/seqid=0 2024-11-27T21:35:26,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742077_1253 (size=12001) 2024-11-27T21:35:26,796 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/d7765506bb4148a38eff6b4b1260ad90 2024-11-27T21:35:26,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:26,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743386790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:26,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:26,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:26,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:26,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:26,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60132 deadline: 1732743386790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:26,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:26,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743386790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:26,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:26,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:26,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:26,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:26,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743386794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:26,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:26,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743386794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:26,799 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:26,799 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:26,799 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:26,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:26,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:26,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:26,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:26,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:26,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:26,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/1d750cadfef7471bb1cb55813e54a68e is 50, key is test_row_0/B:col10/1732743326758/Put/seqid=0 2024-11-27T21:35:26,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742078_1254 (size=12001) 2024-11-27T21:35:26,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-27T21:35:26,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-27T21:35:26,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:26,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. as already flushing 2024-11-27T21:35:26,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:26,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:26,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:26,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:26,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:26,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743386903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:26,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:26,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:26,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:26,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:26,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743386904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:26,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:26,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:26,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:26,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:26,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60132 deadline: 1732743386904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:26,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:26,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:26,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:26,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:26,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743386904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:26,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:26,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:26,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:26,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:26,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743386904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:26,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:26,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:26,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:27,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-27T21:35:27,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:27,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. as already flushing 2024-11-27T21:35:27,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:27,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:27,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-27T21:35:27,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743387114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:27,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:27,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743387114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:27,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:27,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:27,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:27,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:27,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60132 deadline: 1732743387114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:27,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:27,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:27,118 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:27,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:27,118 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743387114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:27,118 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743387114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:27,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:27,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:27,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:27,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:27,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:27,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:27,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-27T21:35:27,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:27,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. as already flushing 2024-11-27T21:35:27,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:27,207 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:27,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:27,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:27,241 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/1d750cadfef7471bb1cb55813e54a68e 2024-11-27T21:35:27,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/82e1771ea7764d6b85e9438f63ed42bd is 50, key is test_row_0/C:col10/1732743326758/Put/seqid=0 2024-11-27T21:35:27,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742079_1255 (size=12001) 2024-11-27T21:35:27,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-27T21:35:27,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:27,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. as already flushing 2024-11-27T21:35:27,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:27,361 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:27,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:27,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:27,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-27T21:35:27,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:27,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743387424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:27,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:27,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:27,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:27,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743387424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:27,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:27,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:27,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:27,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:27,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60132 deadline: 1732743387424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:27,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:27,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:27,427 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:27,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743387424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:27,427 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:27,427 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:27,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:27,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:27,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:27,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:27,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743387424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:27,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:27,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:27,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:27,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-27T21:35:27,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:27,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. as already flushing 2024-11-27T21:35:27,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:27,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:27,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:27,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:27,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-27T21:35:27,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:27,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. as already flushing 2024-11-27T21:35:27,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:27,668 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:27,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:27,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:27,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/82e1771ea7764d6b85e9438f63ed42bd 2024-11-27T21:35:27,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/d7765506bb4148a38eff6b4b1260ad90 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/d7765506bb4148a38eff6b4b1260ad90 2024-11-27T21:35:27,679 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/d7765506bb4148a38eff6b4b1260ad90, entries=150, sequenceid=14, filesize=11.7 K 2024-11-27T21:35:27,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/1d750cadfef7471bb1cb55813e54a68e as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/1d750cadfef7471bb1cb55813e54a68e 2024-11-27T21:35:27,687 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/1d750cadfef7471bb1cb55813e54a68e, entries=150, sequenceid=14, filesize=11.7 K 2024-11-27T21:35:27,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/82e1771ea7764d6b85e9438f63ed42bd as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/82e1771ea7764d6b85e9438f63ed42bd 2024-11-27T21:35:27,692 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/82e1771ea7764d6b85e9438f63ed42bd, entries=150, sequenceid=14, filesize=11.7 K 2024-11-27T21:35:27,693 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for d9a1bb9b05a472888bd945369e4af06a in 935ms, sequenceid=14, compaction requested=false 2024-11-27T21:35:27,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:27,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-27T21:35:27,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:27,821 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing d9a1bb9b05a472888bd945369e4af06a 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-27T21:35:27,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/38bcab1c2a044387962cba88c8b9a3bc is 50, key is test_row_0/A:col10/1732743326789/Put/seqid=0 2024-11-27T21:35:27,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742080_1256 (size=12001) 2024-11-27T21:35:27,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-27T21:35:27,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on d9a1bb9b05a472888bd945369e4af06a 2024-11-27T21:35:27,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. as already flushing 2024-11-27T21:35:27,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:27,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743387942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:27,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:27,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:27,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60132 deadline: 1732743387943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:27,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:27,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:27,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:27,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743387943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:27,947 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:27,947 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:27,947 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:27,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:27,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:27,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:27,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:27,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743387944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:27,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:27,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:27,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:27,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:27,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743387947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:27,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:27,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:27,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:28,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:28,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743388053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:28,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:28,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743388053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:28,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:28,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:28,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:28,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:28,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:28,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:28,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743388053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:28,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:28,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:28,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:28,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743388053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:28,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:28,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:28,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:28,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:28,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:28,237 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/38bcab1c2a044387962cba88c8b9a3bc 2024-11-27T21:35:28,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/dc681db814a442f594705b40596c2a74 is 50, key is test_row_0/B:col10/1732743326789/Put/seqid=0 2024-11-27T21:35:28,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742081_1257 (size=12001) 2024-11-27T21:35:28,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:28,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743388263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:28,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:28,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:28,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:28,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:28,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743388263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:28,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:28,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:28,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743388263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:28,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:28,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743388263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:28,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:28,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:28,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:28,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:28,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:28,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:28,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:28,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:28,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:28,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743388574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:28,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:28,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:28,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:28,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743388574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:28,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:28,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:28,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:28,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:28,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743388574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:28,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:28,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:28,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:28,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:28,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743388574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:28,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:28,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:28,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:28,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:28,651 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/dc681db814a442f594705b40596c2a74 2024-11-27T21:35:28,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/7f005a32ef4240ceb1d3d43eeb97476a is 50, key is test_row_0/C:col10/1732743326789/Put/seqid=0 2024-11-27T21:35:28,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742082_1258 (size=12001) 2024-11-27T21:35:28,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-27T21:35:28,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:28,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60132 deadline: 1732743388963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:28,966 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,39097,1732743267048 for 'test_row_0' in d9a1bb9b05a472888bd945369e4af06a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:35:28,967 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:28,967 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:28,967 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:29,069 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/7f005a32ef4240ceb1d3d43eeb97476a 2024-11-27T21:35:29,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/38bcab1c2a044387962cba88c8b9a3bc as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/38bcab1c2a044387962cba88c8b9a3bc 2024-11-27T21:35:29,078 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/38bcab1c2a044387962cba88c8b9a3bc, entries=150, sequenceid=37, filesize=11.7 K 2024-11-27T21:35:29,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/dc681db814a442f594705b40596c2a74 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/dc681db814a442f594705b40596c2a74 2024-11-27T21:35:29,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:29,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743389083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:29,084 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/dc681db814a442f594705b40596c2a74, entries=150, sequenceid=37, filesize=11.7 K 2024-11-27T21:35:29,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:29,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:29,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:29,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:29,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743389083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:29,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:29,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:29,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:29,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:29,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743389083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:29,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:29,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:29,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:29,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743389083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:29,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:29,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/7f005a32ef4240ceb1d3d43eeb97476a as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/7f005a32ef4240ceb1d3d43eeb97476a 2024-11-27T21:35:29,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:29,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:29,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:29,090 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/7f005a32ef4240ceb1d3d43eeb97476a, entries=150, sequenceid=37, filesize=11.7 K 2024-11-27T21:35:29,091 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for d9a1bb9b05a472888bd945369e4af06a in 1270ms, sequenceid=37, compaction requested=false 2024-11-27T21:35:29,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:29,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:29,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-27T21:35:29,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-27T21:35:29,094 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-27T21:35:29,094 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3450 sec 2024-11-27T21:35:29,096 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.3530 sec 2024-11-27T21:35:29,696 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T21:35:30,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on d9a1bb9b05a472888bd945369e4af06a 2024-11-27T21:35:30,096 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:35:30,096 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d9a1bb9b05a472888bd945369e4af06a 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-27T21:35:30,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/27b0a04af5794db48bf808090b614ab7 is 50, key is test_row_0/A:col10/1732743327941/Put/seqid=0 2024-11-27T21:35:30,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742083_1259 (size=14341) 2024-11-27T21:35:30,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:30,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743390118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:30,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:30,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:30,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:30,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743390120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:30,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:30,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:30,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:30,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:30,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:30,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743390121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:30,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:30,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:30,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:30,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743390122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:30,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:30,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:30,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:30,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:30,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:30,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743390233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:30,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:30,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:30,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:30,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743390233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:30,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:30,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:30,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:30,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743390233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:30,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:30,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:30,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:30,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:30,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:30,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:30,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743390233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:30,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:30,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:30,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:30,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:30,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743390443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:30,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:30,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743390443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:30,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:30,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:30,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:30,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:30,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:30,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743390443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:30,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:30,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:30,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:30,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:30,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:30,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:30,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743390443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:30,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:30,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:30,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:30,509 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/27b0a04af5794db48bf808090b614ab7 2024-11-27T21:35:30,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/ea0fe727a732402fa92a3c1cebf15ea5 is 50, key is test_row_0/B:col10/1732743327941/Put/seqid=0 2024-11-27T21:35:30,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742084_1260 (size=12001) 2024-11-27T21:35:30,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:30,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743390753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:30,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:30,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:30,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:30,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:30,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743390754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:30,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:30,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:30,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:30,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:30,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743390754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:30,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:30,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:30,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:30,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:30,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743390754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:30,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:30,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:30,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:30,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-27T21:35:30,913 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:35:30,914 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:35:30,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-27T21:35:30,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-27T21:35:30,916 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:35:30,917 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:35:30,917 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:35:30,922 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/ea0fe727a732402fa92a3c1cebf15ea5 2024-11-27T21:35:30,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/7b61fd7b494a48f6ac179a1f063419a0 is 50, key is test_row_0/C:col10/1732743327941/Put/seqid=0 2024-11-27T21:35:30,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742085_1261 (size=12001) 2024-11-27T21:35:30,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:30,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60132 deadline: 1732743390984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:30,987 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,39097,1732743267048 for 'test_row_0' in d9a1bb9b05a472888bd945369e4af06a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:35:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:31,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-27T21:35:31,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-27T21:35:31,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:31,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. as already flushing 2024-11-27T21:35:31,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:31,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:31,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:31,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:31,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-27T21:35:31,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:31,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. as already flushing 2024-11-27T21:35:31,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:31,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:31,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:31,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:31,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-27T21:35:31,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:31,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743391263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:31,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:31,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:31,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:31,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:31,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743391263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:31,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:31,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:31,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:31,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:31,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743391263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:31,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:31,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:31,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:31,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743391263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:31,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:31,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:31,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:31,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:31,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/7b61fd7b494a48f6ac179a1f063419a0 2024-11-27T21:35:31,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/27b0a04af5794db48bf808090b614ab7 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/27b0a04af5794db48bf808090b614ab7 2024-11-27T21:35:31,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/27b0a04af5794db48bf808090b614ab7, entries=200, sequenceid=53, filesize=14.0 K 2024-11-27T21:35:31,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/ea0fe727a732402fa92a3c1cebf15ea5 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/ea0fe727a732402fa92a3c1cebf15ea5 2024-11-27T21:35:31,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/ea0fe727a732402fa92a3c1cebf15ea5, entries=150, sequenceid=53, filesize=11.7 K 2024-11-27T21:35:31,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/7b61fd7b494a48f6ac179a1f063419a0 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/7b61fd7b494a48f6ac179a1f063419a0 2024-11-27T21:35:31,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/7b61fd7b494a48f6ac179a1f063419a0, entries=150, sequenceid=53, filesize=11.7 K 2024-11-27T21:35:31,354 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for d9a1bb9b05a472888bd945369e4af06a in 1258ms, sequenceid=53, compaction requested=true 2024-11-27T21:35:31,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:31,354 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:35:31,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a1bb9b05a472888bd945369e4af06a:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:35:31,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:31,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a1bb9b05a472888bd945369e4af06a:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:35:31,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:31,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a1bb9b05a472888bd945369e4af06a:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:35:31,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:35:31,354 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:35:31,355 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:35:31,356 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): d9a1bb9b05a472888bd945369e4af06a/A is initiating minor compaction (all files) 2024-11-27T21:35:31,356 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a1bb9b05a472888bd945369e4af06a/A in TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:31,356 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/d7765506bb4148a38eff6b4b1260ad90, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/38bcab1c2a044387962cba88c8b9a3bc, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/27b0a04af5794db48bf808090b614ab7] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp, totalSize=37.4 K 2024-11-27T21:35:31,356 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:35:31,356 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): d9a1bb9b05a472888bd945369e4af06a/B is initiating minor compaction (all files) 2024-11-27T21:35:31,356 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a1bb9b05a472888bd945369e4af06a/B in TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:31,356 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/1d750cadfef7471bb1cb55813e54a68e, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/dc681db814a442f594705b40596c2a74, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/ea0fe727a732402fa92a3c1cebf15ea5] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp, totalSize=35.2 K 2024-11-27T21:35:31,356 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7765506bb4148a38eff6b4b1260ad90, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732743326755 2024-11-27T21:35:31,357 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d750cadfef7471bb1cb55813e54a68e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732743326755 2024-11-27T21:35:31,357 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38bcab1c2a044387962cba88c8b9a3bc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732743326788 2024-11-27T21:35:31,358 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting dc681db814a442f594705b40596c2a74, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732743326788 2024-11-27T21:35:31,358 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27b0a04af5794db48bf808090b614ab7, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732743327941 2024-11-27T21:35:31,358 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting ea0fe727a732402fa92a3c1cebf15ea5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732743327941 2024-11-27T21:35:31,367 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a1bb9b05a472888bd945369e4af06a#A#compaction#217 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:31,368 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/4df7da5891e34aa6a9354358985286d3 is 50, key is test_row_0/A:col10/1732743327941/Put/seqid=0 2024-11-27T21:35:31,370 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a1bb9b05a472888bd945369e4af06a#B#compaction#218 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:31,371 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/9b0dfa0f5065461eb3524c922bf6db3d is 50, key is test_row_0/B:col10/1732743327941/Put/seqid=0 2024-11-27T21:35:31,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742087_1263 (size=12104) 2024-11-27T21:35:31,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-27T21:35:31,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:31,376 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing d9a1bb9b05a472888bd945369e4af06a 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-27T21:35:31,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742086_1262 (size=12104) 2024-11-27T21:35:31,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/a4161037ddbf40ff84096e4b6562700b is 50, key is test_row_0/A:col10/1732743330119/Put/seqid=0 2024-11-27T21:35:31,387 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/9b0dfa0f5065461eb3524c922bf6db3d as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/9b0dfa0f5065461eb3524c922bf6db3d 2024-11-27T21:35:31,388 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/4df7da5891e34aa6a9354358985286d3 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/4df7da5891e34aa6a9354358985286d3 2024-11-27T21:35:31,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742088_1264 (size=12001) 2024-11-27T21:35:31,393 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9a1bb9b05a472888bd945369e4af06a/B of d9a1bb9b05a472888bd945369e4af06a into 9b0dfa0f5065461eb3524c922bf6db3d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:31,394 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/a4161037ddbf40ff84096e4b6562700b 2024-11-27T21:35:31,394 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:31,394 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., storeName=d9a1bb9b05a472888bd945369e4af06a/B, priority=13, startTime=1732743331354; duration=0sec 2024-11-27T21:35:31,394 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:35:31,394 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a1bb9b05a472888bd945369e4af06a:B 2024-11-27T21:35:31,394 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:35:31,396 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:35:31,396 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): d9a1bb9b05a472888bd945369e4af06a/C is initiating minor compaction (all files) 2024-11-27T21:35:31,396 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a1bb9b05a472888bd945369e4af06a/C in TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:31,397 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/82e1771ea7764d6b85e9438f63ed42bd, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/7f005a32ef4240ceb1d3d43eeb97476a, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/7b61fd7b494a48f6ac179a1f063419a0] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp, totalSize=35.2 K 2024-11-27T21:35:31,397 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9a1bb9b05a472888bd945369e4af06a/A of d9a1bb9b05a472888bd945369e4af06a into 4df7da5891e34aa6a9354358985286d3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:31,397 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:31,397 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., storeName=d9a1bb9b05a472888bd945369e4af06a/A, priority=13, startTime=1732743331354; duration=0sec 2024-11-27T21:35:31,397 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:31,397 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a1bb9b05a472888bd945369e4af06a:A 2024-11-27T21:35:31,397 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 82e1771ea7764d6b85e9438f63ed42bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732743326755 2024-11-27T21:35:31,398 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f005a32ef4240ceb1d3d43eeb97476a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732743326788 2024-11-27T21:35:31,398 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b61fd7b494a48f6ac179a1f063419a0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732743327941 2024-11-27T21:35:31,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/f793410708a6497faec41231722caa11 is 50, key is test_row_0/B:col10/1732743330119/Put/seqid=0 2024-11-27T21:35:31,408 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a1bb9b05a472888bd945369e4af06a#C#compaction#221 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:31,408 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/3c8093c213324d66ae9ce1f8a8b069a6 is 50, key is test_row_0/C:col10/1732743327941/Put/seqid=0 2024-11-27T21:35:31,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742089_1265 (size=12001) 2024-11-27T21:35:31,414 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/f793410708a6497faec41231722caa11 2024-11-27T21:35:31,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/a7eb42d75bb2430c811170f31e6e5e21 is 50, key is test_row_0/C:col10/1732743330119/Put/seqid=0 2024-11-27T21:35:31,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742090_1266 (size=12104) 2024-11-27T21:35:31,440 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/3c8093c213324d66ae9ce1f8a8b069a6 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/3c8093c213324d66ae9ce1f8a8b069a6 2024-11-27T21:35:31,448 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9a1bb9b05a472888bd945369e4af06a/C of d9a1bb9b05a472888bd945369e4af06a into 3c8093c213324d66ae9ce1f8a8b069a6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:31,448 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:31,448 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., storeName=d9a1bb9b05a472888bd945369e4af06a/C, priority=13, startTime=1732743331354; duration=0sec 2024-11-27T21:35:31,448 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:31,448 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a1bb9b05a472888bd945369e4af06a:C 2024-11-27T21:35:31,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742091_1267 (size=12001) 2024-11-27T21:35:31,467 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/a7eb42d75bb2430c811170f31e6e5e21 2024-11-27T21:35:31,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/a4161037ddbf40ff84096e4b6562700b as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/a4161037ddbf40ff84096e4b6562700b 2024-11-27T21:35:31,478 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/a4161037ddbf40ff84096e4b6562700b, entries=150, sequenceid=73, filesize=11.7 K 2024-11-27T21:35:31,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/f793410708a6497faec41231722caa11 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/f793410708a6497faec41231722caa11 2024-11-27T21:35:31,484 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/f793410708a6497faec41231722caa11, entries=150, sequenceid=73, filesize=11.7 K 2024-11-27T21:35:31,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/a7eb42d75bb2430c811170f31e6e5e21 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/a7eb42d75bb2430c811170f31e6e5e21 2024-11-27T21:35:31,496 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/a7eb42d75bb2430c811170f31e6e5e21, entries=150, sequenceid=73, filesize=11.7 K 2024-11-27T21:35:31,497 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=0 B/0 for d9a1bb9b05a472888bd945369e4af06a in 121ms, sequenceid=73, compaction requested=false 2024-11-27T21:35:31,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:31,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:31,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-27T21:35:31,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-27T21:35:31,503 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-27T21:35:31,503 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 582 msec 2024-11-27T21:35:31,506 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 591 msec 2024-11-27T21:35:31,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-27T21:35:31,543 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:35:31,544 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:35:31,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-27T21:35:31,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-27T21:35:31,546 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:35:31,547 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:35:31,547 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:35:31,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-27T21:35:31,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-27T21:35:31,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:31,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:31,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:31,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-27T21:35:31,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-27T21:35:31,704 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-27T21:35:31,705 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-11-27T21:35:31,707 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 161 msec 2024-11-27T21:35:31,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-27T21:35:31,864 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:35:31,864 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:35:31,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-27T21:35:31,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-27T21:35:31,866 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:35:31,867 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:35:31,867 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:35:31,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-27T21:35:32,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-27T21:35:32,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:32,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:32,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:32,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-27T21:35:32,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-27T21:35:32,024 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-27T21:35:32,024 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-11-27T21:35:32,026 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 161 msec 2024-11-27T21:35:32,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-27T21:35:32,184 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:35:32,185 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:35:32,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-27T21:35:32,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-27T21:35:32,187 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:35:32,188 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:35:32,188 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:35:32,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on d9a1bb9b05a472888bd945369e4af06a 2024-11-27T21:35:32,286 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:35:32,287 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d9a1bb9b05a472888bd945369e4af06a 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T21:35:32,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-27T21:35:32,293 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/8145be8efda04281a8a615d4b194e1ed is 50, key is test_row_0/A:col10/1732743332287/Put/seqid=0 2024-11-27T21:35:32,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742092_1268 (size=14341) 2024-11-27T21:35:32,298 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/8145be8efda04281a8a615d4b194e1ed 2024-11-27T21:35:32,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/d77aaa06fb2449a18752ad280db78f95 is 50, key is test_row_0/B:col10/1732743332287/Put/seqid=0 2024-11-27T21:35:32,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742093_1269 (size=12001) 2024-11-27T21:35:32,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:32,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743392315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:32,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:32,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:32,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:32,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743392315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:32,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:32,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:32,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:32,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:32,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:32,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743392319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:32,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:32,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:32,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:32,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:32,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743392320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:32,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:32,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:32,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:32,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-27T21:35:32,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:32,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. as already flushing 2024-11-27T21:35:32,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:32,341 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:32,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:32,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:32,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:32,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743392423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:32,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:32,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:32,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:32,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743392423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:32,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:32,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:32,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:32,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:32,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:32,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743392433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:32,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:32,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:32,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743392433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:32,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:32,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:32,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:32,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:32,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:32,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-27T21:35:32,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:32,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. as already flushing 2024-11-27T21:35:32,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:32,495 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:32,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:32,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:32,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-27T21:35:32,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:32,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743392633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:32,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:32,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:32,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:32,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743392633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:32,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:32,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:32,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:32,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:32,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:32,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743392643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:32,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:32,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743392643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:32,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:32,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:32,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:32,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:32,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:32,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:32,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-27T21:35:32,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:32,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. as already flushing 2024-11-27T21:35:32,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:32,648 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:32,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:32,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:32,712 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/d77aaa06fb2449a18752ad280db78f95 2024-11-27T21:35:32,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/f1be662485e14b04ae62a375f64ddeff is 50, key is test_row_0/C:col10/1732743332287/Put/seqid=0 2024-11-27T21:35:32,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742094_1270 (size=12001) 2024-11-27T21:35:32,724 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/f1be662485e14b04ae62a375f64ddeff 2024-11-27T21:35:32,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/8145be8efda04281a8a615d4b194e1ed as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/8145be8efda04281a8a615d4b194e1ed 2024-11-27T21:35:32,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/8145be8efda04281a8a615d4b194e1ed, entries=200, sequenceid=90, filesize=14.0 K 2024-11-27T21:35:32,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/d77aaa06fb2449a18752ad280db78f95 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/d77aaa06fb2449a18752ad280db78f95 2024-11-27T21:35:32,737 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/d77aaa06fb2449a18752ad280db78f95, entries=150, sequenceid=90, filesize=11.7 K 2024-11-27T21:35:32,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/f1be662485e14b04ae62a375f64ddeff as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/f1be662485e14b04ae62a375f64ddeff 2024-11-27T21:35:32,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/f1be662485e14b04ae62a375f64ddeff, entries=150, sequenceid=90, filesize=11.7 K 2024-11-27T21:35:32,742 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for d9a1bb9b05a472888bd945369e4af06a in 456ms, sequenceid=90, compaction requested=true 2024-11-27T21:35:32,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:32,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a1bb9b05a472888bd945369e4af06a:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:35:32,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:32,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a1bb9b05a472888bd945369e4af06a:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:35:32,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:32,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a1bb9b05a472888bd945369e4af06a:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:35:32,742 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:35:32,742 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:35:32,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:35:32,743 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:35:32,744 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): d9a1bb9b05a472888bd945369e4af06a/A is initiating minor compaction (all files) 2024-11-27T21:35:32,744 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a1bb9b05a472888bd945369e4af06a/A in TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:32,744 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/4df7da5891e34aa6a9354358985286d3, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/a4161037ddbf40ff84096e4b6562700b, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/8145be8efda04281a8a615d4b194e1ed] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp, totalSize=37.5 K 2024-11-27T21:35:32,744 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:35:32,744 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): d9a1bb9b05a472888bd945369e4af06a/B is initiating minor compaction (all files) 2024-11-27T21:35:32,744 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a1bb9b05a472888bd945369e4af06a/B in TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:32,744 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4df7da5891e34aa6a9354358985286d3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732743327941 2024-11-27T21:35:32,744 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/9b0dfa0f5065461eb3524c922bf6db3d, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/f793410708a6497faec41231722caa11, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/d77aaa06fb2449a18752ad280db78f95] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp, totalSize=35.3 K 2024-11-27T21:35:32,745 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4161037ddbf40ff84096e4b6562700b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1732743330117 2024-11-27T21:35:32,745 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b0dfa0f5065461eb3524c922bf6db3d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732743327941 2024-11-27T21:35:32,745 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8145be8efda04281a8a615d4b194e1ed, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732743332281 2024-11-27T21:35:32,745 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting f793410708a6497faec41231722caa11, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1732743330117 2024-11-27T21:35:32,746 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting d77aaa06fb2449a18752ad280db78f95, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732743332281 2024-11-27T21:35:32,765 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a1bb9b05a472888bd945369e4af06a#B#compaction#226 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:32,765 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a1bb9b05a472888bd945369e4af06a#A#compaction#227 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:32,765 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/9fca3a3c2c464cfb9693d4edae0f2e32 is 50, key is test_row_0/B:col10/1732743332287/Put/seqid=0 2024-11-27T21:35:32,766 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/0726c1d6bafe4772a99f78a875161ea5 is 50, key is test_row_0/A:col10/1732743332287/Put/seqid=0 2024-11-27T21:35:32,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742095_1271 (size=12207) 2024-11-27T21:35:32,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742096_1272 (size=12207) 2024-11-27T21:35:32,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-27T21:35:32,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:32,801 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing d9a1bb9b05a472888bd945369e4af06a 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-27T21:35:32,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/ad21ab174ef64322b62aaeebfc7bc0da is 50, key is test_row_0/A:col10/1732743332318/Put/seqid=0 2024-11-27T21:35:32,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742097_1273 (size=12001) 2024-11-27T21:35:32,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-27T21:35:32,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on d9a1bb9b05a472888bd945369e4af06a 2024-11-27T21:35:32,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. as already flushing 2024-11-27T21:35:32,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:32,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743392958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:32,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:32,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743392959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:32,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:32,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:32,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:32,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:32,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:32,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:32,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:32,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743392963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:32,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:32,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:32,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:32,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743392963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:32,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:32,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:32,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:32,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:33,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:33,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743393073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:33,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:33,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:33,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:33,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:33,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743393073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:33,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:33,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743393073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:33,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:33,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:33,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:33,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:33,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:33,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:33,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:33,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743393073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:33,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:33,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:33,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:33,176 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/9fca3a3c2c464cfb9693d4edae0f2e32 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/9fca3a3c2c464cfb9693d4edae0f2e32 2024-11-27T21:35:33,179 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/0726c1d6bafe4772a99f78a875161ea5 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/0726c1d6bafe4772a99f78a875161ea5 2024-11-27T21:35:33,182 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9a1bb9b05a472888bd945369e4af06a/B of d9a1bb9b05a472888bd945369e4af06a into 9fca3a3c2c464cfb9693d4edae0f2e32(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:33,182 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:33,182 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., storeName=d9a1bb9b05a472888bd945369e4af06a/B, priority=13, startTime=1732743332742; duration=0sec 2024-11-27T21:35:33,182 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:35:33,182 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a1bb9b05a472888bd945369e4af06a:B 2024-11-27T21:35:33,182 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:35:33,184 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:35:33,185 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): d9a1bb9b05a472888bd945369e4af06a/C is initiating minor compaction (all files) 2024-11-27T21:35:33,185 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a1bb9b05a472888bd945369e4af06a/C in TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:33,185 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/3c8093c213324d66ae9ce1f8a8b069a6, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/a7eb42d75bb2430c811170f31e6e5e21, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/f1be662485e14b04ae62a375f64ddeff] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp, totalSize=35.3 K 2024-11-27T21:35:33,185 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c8093c213324d66ae9ce1f8a8b069a6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732743327941 2024-11-27T21:35:33,186 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting a7eb42d75bb2430c811170f31e6e5e21, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1732743330117 2024-11-27T21:35:33,186 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting f1be662485e14b04ae62a375f64ddeff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732743332281 2024-11-27T21:35:33,186 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9a1bb9b05a472888bd945369e4af06a/A of d9a1bb9b05a472888bd945369e4af06a into 0726c1d6bafe4772a99f78a875161ea5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:33,186 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:33,186 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., storeName=d9a1bb9b05a472888bd945369e4af06a/A, priority=13, startTime=1732743332742; duration=0sec 2024-11-27T21:35:33,186 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:33,187 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a1bb9b05a472888bd945369e4af06a:A 2024-11-27T21:35:33,192 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a1bb9b05a472888bd945369e4af06a#C#compaction#229 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:33,193 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/eb8f04fca954401a8eb1f9c0b4099853 is 50, key is test_row_0/C:col10/1732743332287/Put/seqid=0 2024-11-27T21:35:33,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742098_1274 (size=12207) 2024-11-27T21:35:33,211 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/ad21ab174ef64322b62aaeebfc7bc0da 2024-11-27T21:35:33,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/ee6563bd37784e308cecf445842df054 is 50, key is test_row_0/B:col10/1732743332318/Put/seqid=0 2024-11-27T21:35:33,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742099_1275 (size=12001) 2024-11-27T21:35:33,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:33,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743393283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:33,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:33,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743393283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:33,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:33,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:33,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:33,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:33,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:33,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:33,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743393283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:33,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:33,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:33,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:33,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:33,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:33,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743393283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:33,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:33,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:33,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:33,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-27T21:35:33,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:33,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743393594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:33,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:33,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:33,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:33,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:33,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743393594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:33,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:33,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:33,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:33,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:33,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743393594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:33,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:33,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:33,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743393594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:33,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:33,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:33,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:33,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:33,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:33,602 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/eb8f04fca954401a8eb1f9c0b4099853 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/eb8f04fca954401a8eb1f9c0b4099853 2024-11-27T21:35:33,607 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9a1bb9b05a472888bd945369e4af06a/C of d9a1bb9b05a472888bd945369e4af06a into eb8f04fca954401a8eb1f9c0b4099853(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:33,607 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:33,607 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., storeName=d9a1bb9b05a472888bd945369e4af06a/C, priority=13, startTime=1732743332742; duration=0sec 2024-11-27T21:35:33,607 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:33,608 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a1bb9b05a472888bd945369e4af06a:C 2024-11-27T21:35:33,623 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/ee6563bd37784e308cecf445842df054 2024-11-27T21:35:33,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/7667452d652b48688148a1a858751c29 is 50, key is test_row_0/C:col10/1732743332318/Put/seqid=0 2024-11-27T21:35:33,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742100_1276 (size=12001) 2024-11-27T21:35:33,635 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/7667452d652b48688148a1a858751c29 2024-11-27T21:35:33,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/ad21ab174ef64322b62aaeebfc7bc0da as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/ad21ab174ef64322b62aaeebfc7bc0da 2024-11-27T21:35:33,643 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/ad21ab174ef64322b62aaeebfc7bc0da, entries=150, sequenceid=114, filesize=11.7 K 2024-11-27T21:35:33,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/ee6563bd37784e308cecf445842df054 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/ee6563bd37784e308cecf445842df054 2024-11-27T21:35:33,649 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/ee6563bd37784e308cecf445842df054, entries=150, sequenceid=114, filesize=11.7 K 2024-11-27T21:35:33,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/7667452d652b48688148a1a858751c29 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/7667452d652b48688148a1a858751c29 2024-11-27T21:35:33,653 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/7667452d652b48688148a1a858751c29, entries=150, sequenceid=114, filesize=11.7 K 2024-11-27T21:35:33,654 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for d9a1bb9b05a472888bd945369e4af06a in 853ms, sequenceid=114, compaction requested=false 2024-11-27T21:35:33,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:33,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:33,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-27T21:35:33,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-27T21:35:33,658 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-27T21:35:33,658 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4680 sec 2024-11-27T21:35:33,660 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.4740 sec 2024-11-27T21:35:34,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on d9a1bb9b05a472888bd945369e4af06a 2024-11-27T21:35:34,113 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:35:34,113 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d9a1bb9b05a472888bd945369e4af06a 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-27T21:35:34,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/722d09fb152145449760b3ed1a399a66 is 50, key is test_row_0/A:col10/1732743334105/Put/seqid=0 2024-11-27T21:35:34,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742101_1277 (size=14541) 2024-11-27T21:35:34,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:34,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743394139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:34,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:34,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:34,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:34,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:34,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743394142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:34,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:34,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:34,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:34,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:34,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743394144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:34,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:34,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:34,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:34,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743394145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:34,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:34,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:34,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:34,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:34,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:34,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743394254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:34,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:34,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743394254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:34,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:34,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:34,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:34,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743394254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:34,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:34,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:34,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:34,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:34,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:34,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:34,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:34,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:34,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743394254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:34,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:34,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:34,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:34,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-27T21:35:34,333 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:35:34,333 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:35:34,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-27T21:35:34,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-27T21:35:34,335 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:35:34,336 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:35:34,336 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:35:34,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-27T21:35:34,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:34,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743394463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:34,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:34,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:34,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:34,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:34,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743394463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:34,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:34,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:34,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:34,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:34,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743394463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:34,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:34,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:34,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:34,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:34,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743394463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:34,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:34,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:34,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:34,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-27T21:35:34,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:34,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. as already flushing 2024-11-27T21:35:34,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:34,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:34,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:34,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:34,535 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/722d09fb152145449760b3ed1a399a66 2024-11-27T21:35:34,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/6e27d578c74f4189af1ac5188a42df63 is 50, key is test_row_0/B:col10/1732743334105/Put/seqid=0 2024-11-27T21:35:34,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742102_1278 (size=12151) 2024-11-27T21:35:34,572 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/6e27d578c74f4189af1ac5188a42df63 2024-11-27T21:35:34,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/c2c5dfef8e9643dd91337635c730735a is 50, key is test_row_0/C:col10/1732743334105/Put/seqid=0 2024-11-27T21:35:34,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742103_1279 (size=12151) 2024-11-27T21:35:34,599 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/c2c5dfef8e9643dd91337635c730735a 2024-11-27T21:35:34,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/722d09fb152145449760b3ed1a399a66 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/722d09fb152145449760b3ed1a399a66 2024-11-27T21:35:34,611 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/722d09fb152145449760b3ed1a399a66, entries=200, sequenceid=134, filesize=14.2 K 2024-11-27T21:35:34,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/6e27d578c74f4189af1ac5188a42df63 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/6e27d578c74f4189af1ac5188a42df63 2024-11-27T21:35:34,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/6e27d578c74f4189af1ac5188a42df63, entries=150, sequenceid=134, filesize=11.9 K 2024-11-27T21:35:34,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/c2c5dfef8e9643dd91337635c730735a as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/c2c5dfef8e9643dd91337635c730735a 2024-11-27T21:35:34,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/c2c5dfef8e9643dd91337635c730735a, entries=150, sequenceid=134, filesize=11.9 K 2024-11-27T21:35:34,631 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for d9a1bb9b05a472888bd945369e4af06a in 518ms, sequenceid=134, compaction requested=true 2024-11-27T21:35:34,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:34,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a1bb9b05a472888bd945369e4af06a:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:35:34,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:34,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a1bb9b05a472888bd945369e4af06a:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:35:34,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:34,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a1bb9b05a472888bd945369e4af06a:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:35:34,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:35:34,631 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:35:34,631 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:35:34,632 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38749 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:35:34,632 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): d9a1bb9b05a472888bd945369e4af06a/A is initiating minor compaction (all files) 2024-11-27T21:35:34,632 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a1bb9b05a472888bd945369e4af06a/A in TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:34,633 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/0726c1d6bafe4772a99f78a875161ea5, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/ad21ab174ef64322b62aaeebfc7bc0da, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/722d09fb152145449760b3ed1a399a66] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp, totalSize=37.8 K 2024-11-27T21:35:34,633 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:35:34,633 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): d9a1bb9b05a472888bd945369e4af06a/B is initiating minor compaction (all files) 2024-11-27T21:35:34,633 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a1bb9b05a472888bd945369e4af06a/B in TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:34,633 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/9fca3a3c2c464cfb9693d4edae0f2e32, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/ee6563bd37784e308cecf445842df054, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/6e27d578c74f4189af1ac5188a42df63] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp, totalSize=35.5 K 2024-11-27T21:35:34,634 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fca3a3c2c464cfb9693d4edae0f2e32, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732743332281 2024-11-27T21:35:34,634 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0726c1d6bafe4772a99f78a875161ea5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732743332281 2024-11-27T21:35:34,634 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting ee6563bd37784e308cecf445842df054, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732743332305 2024-11-27T21:35:34,634 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad21ab174ef64322b62aaeebfc7bc0da, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732743332305 2024-11-27T21:35:34,634 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e27d578c74f4189af1ac5188a42df63, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732743334105 2024-11-27T21:35:34,634 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 722d09fb152145449760b3ed1a399a66, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732743334105 2024-11-27T21:35:34,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-27T21:35:34,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:34,644 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing d9a1bb9b05a472888bd945369e4af06a 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-27T21:35:34,647 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a1bb9b05a472888bd945369e4af06a#A#compaction#235 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:34,648 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/50e03acc026743a092d7d87aacfe80e6 is 50, key is test_row_0/A:col10/1732743334105/Put/seqid=0 2024-11-27T21:35:34,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-27T21:35:34,666 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a1bb9b05a472888bd945369e4af06a#B#compaction#236 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:34,667 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/2aa5d24831ab421397dbcbe8cfa4a350 is 50, key is test_row_0/B:col10/1732743334105/Put/seqid=0 2024-11-27T21:35:34,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/b7b34394407a4fcb92018f884903559a is 50, key is test_row_0/A:col10/1732743334143/Put/seqid=0 2024-11-27T21:35:34,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742104_1280 (size=12459) 2024-11-27T21:35:34,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742106_1282 (size=12151) 2024-11-27T21:35:34,707 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/b7b34394407a4fcb92018f884903559a 2024-11-27T21:35:34,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742105_1281 (size=12459) 2024-11-27T21:35:34,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/7482c3589fb54e5ea61626332319485a is 50, key is test_row_0/B:col10/1732743334143/Put/seqid=0 2024-11-27T21:35:34,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742107_1283 (size=12151) 2024-11-27T21:35:34,753 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/7482c3589fb54e5ea61626332319485a 2024-11-27T21:35:34,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. as already flushing 2024-11-27T21:35:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on d9a1bb9b05a472888bd945369e4af06a 2024-11-27T21:35:34,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/26ccb361a1d24b7ab82d5ddf52fb2e68 is 50, key is test_row_0/C:col10/1732743334143/Put/seqid=0 2024-11-27T21:35:34,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:34,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743394793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:34,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:34,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:34,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:34,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:34,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:34,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743394795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:34,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743394795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:34,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:34,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743394797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:34,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:34,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:34,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:34,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:34,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:34,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:34,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:34,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:34,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:34,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742108_1284 (size=12151) 2024-11-27T21:35:34,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:34,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743394903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:34,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:34,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:34,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:34,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743394903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:34,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:34,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:34,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:34,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:34,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:34,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743394914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:34,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:34,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743394915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:34,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:34,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:34,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:34,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:34,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:34,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:34,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-27T21:35:35,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:35,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60132 deadline: 1732743395014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:35,017 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,39097,1732743267048 for 'test_row_0' in d9a1bb9b05a472888bd945369e4af06a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:35:35,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:35,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:35,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:35,098 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/50e03acc026743a092d7d87aacfe80e6 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/50e03acc026743a092d7d87aacfe80e6 2024-11-27T21:35:35,105 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9a1bb9b05a472888bd945369e4af06a/A of d9a1bb9b05a472888bd945369e4af06a into 50e03acc026743a092d7d87aacfe80e6(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:35,105 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:35,105 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., storeName=d9a1bb9b05a472888bd945369e4af06a/A, priority=13, startTime=1732743334631; duration=0sec 2024-11-27T21:35:35,106 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:35:35,106 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a1bb9b05a472888bd945369e4af06a:A 2024-11-27T21:35:35,106 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:35:35,107 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:35:35,107 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): d9a1bb9b05a472888bd945369e4af06a/C is initiating minor compaction (all files) 2024-11-27T21:35:35,107 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a1bb9b05a472888bd945369e4af06a/C in TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:35,107 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/eb8f04fca954401a8eb1f9c0b4099853, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/7667452d652b48688148a1a858751c29, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/c2c5dfef8e9643dd91337635c730735a] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp, totalSize=35.5 K 2024-11-27T21:35:35,108 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb8f04fca954401a8eb1f9c0b4099853, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732743332281 2024-11-27T21:35:35,108 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7667452d652b48688148a1a858751c29, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732743332305 2024-11-27T21:35:35,109 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2c5dfef8e9643dd91337635c730735a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732743334105 2024-11-27T21:35:35,118 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/2aa5d24831ab421397dbcbe8cfa4a350 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/2aa5d24831ab421397dbcbe8cfa4a350 2024-11-27T21:35:35,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:35,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743395113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:35,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:35,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:35,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:35,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:35,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743395113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:35,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:35,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:35,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:35,125 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9a1bb9b05a472888bd945369e4af06a/B of d9a1bb9b05a472888bd945369e4af06a into 2aa5d24831ab421397dbcbe8cfa4a350(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:35,125 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:35,125 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., storeName=d9a1bb9b05a472888bd945369e4af06a/B, priority=13, startTime=1732743334631; duration=0sec 2024-11-27T21:35:35,125 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:35,125 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a1bb9b05a472888bd945369e4af06a:B 2024-11-27T21:35:35,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:35,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743395123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:35,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:35,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743395124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:35,130 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a1bb9b05a472888bd945369e4af06a#C#compaction#240 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:35,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:35,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:35,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:35,131 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/ad8d4284f72544eea73700368529be18 is 50, key is test_row_0/C:col10/1732743334105/Put/seqid=0 2024-11-27T21:35:35,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:35,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:35,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:35,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742109_1285 (size=12459) 2024-11-27T21:35:35,158 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/ad8d4284f72544eea73700368529be18 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/ad8d4284f72544eea73700368529be18 2024-11-27T21:35:35,165 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9a1bb9b05a472888bd945369e4af06a/C of d9a1bb9b05a472888bd945369e4af06a into ad8d4284f72544eea73700368529be18(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:35,165 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:35,165 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., storeName=d9a1bb9b05a472888bd945369e4af06a/C, priority=13, startTime=1732743334631; duration=0sec 2024-11-27T21:35:35,166 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:35,166 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a1bb9b05a472888bd945369e4af06a:C 2024-11-27T21:35:35,204 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/26ccb361a1d24b7ab82d5ddf52fb2e68 2024-11-27T21:35:35,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/b7b34394407a4fcb92018f884903559a as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/b7b34394407a4fcb92018f884903559a 2024-11-27T21:35:35,216 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/b7b34394407a4fcb92018f884903559a, entries=150, sequenceid=154, filesize=11.9 K 2024-11-27T21:35:35,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/7482c3589fb54e5ea61626332319485a as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/7482c3589fb54e5ea61626332319485a 2024-11-27T21:35:35,225 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/7482c3589fb54e5ea61626332319485a, entries=150, sequenceid=154, filesize=11.9 K 2024-11-27T21:35:35,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/26ccb361a1d24b7ab82d5ddf52fb2e68 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/26ccb361a1d24b7ab82d5ddf52fb2e68 2024-11-27T21:35:35,231 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/26ccb361a1d24b7ab82d5ddf52fb2e68, entries=150, sequenceid=154, filesize=11.9 K 2024-11-27T21:35:35,232 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for d9a1bb9b05a472888bd945369e4af06a in 588ms, sequenceid=154, compaction requested=false 2024-11-27T21:35:35,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:35,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:35,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-27T21:35:35,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-27T21:35:35,239 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-27T21:35:35,239 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 900 msec 2024-11-27T21:35:35,243 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 907 msec 2024-11-27T21:35:35,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on d9a1bb9b05a472888bd945369e4af06a 2024-11-27T21:35:35,426 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:35:35,426 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d9a1bb9b05a472888bd945369e4af06a 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-27T21:35:35,432 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/a263947548b54002acecd65192ff7652 is 50, key is test_row_0/A:col10/1732743334791/Put/seqid=0 2024-11-27T21:35:35,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742110_1286 (size=12151) 2024-11-27T21:35:35,439 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/a263947548b54002acecd65192ff7652 2024-11-27T21:35:35,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/c576f6e080094a80ba1a5384c4948823 is 50, key is test_row_0/B:col10/1732743334791/Put/seqid=0 2024-11-27T21:35:35,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742111_1287 (size=12151) 2024-11-27T21:35:35,454 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/c576f6e080094a80ba1a5384c4948823 2024-11-27T21:35:35,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:35,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743395456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:35,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:35,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:35,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:35,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743395456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:35,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:35,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:35,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:35,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:35,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/0513dd2a4cd140dc9a4b42e3472bf730 is 50, key is test_row_0/C:col10/1732743334791/Put/seqid=0 2024-11-27T21:35:35,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:35,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743395462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:35,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:35,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:35,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:35,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743395463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:35,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:35,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:35,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:35,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:35,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742112_1288 (size=12151) 2024-11-27T21:35:35,471 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/0513dd2a4cd140dc9a4b42e3472bf730 2024-11-27T21:35:35,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-27T21:35:35,474 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:35:35,475 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:35:35,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-27T21:35:35,477 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:35:35,478 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:35:35,478 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:35:35,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-27T21:35:35,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/a263947548b54002acecd65192ff7652 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/a263947548b54002acecd65192ff7652 2024-11-27T21:35:35,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/a263947548b54002acecd65192ff7652, entries=150, sequenceid=174, filesize=11.9 K 2024-11-27T21:35:35,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/c576f6e080094a80ba1a5384c4948823 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/c576f6e080094a80ba1a5384c4948823 2024-11-27T21:35:35,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/c576f6e080094a80ba1a5384c4948823, entries=150, sequenceid=174, filesize=11.9 K 2024-11-27T21:35:35,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/0513dd2a4cd140dc9a4b42e3472bf730 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/0513dd2a4cd140dc9a4b42e3472bf730 2024-11-27T21:35:35,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/0513dd2a4cd140dc9a4b42e3472bf730, entries=150, sequenceid=174, filesize=11.9 K 2024-11-27T21:35:35,499 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for d9a1bb9b05a472888bd945369e4af06a in 73ms, sequenceid=174, compaction requested=true 2024-11-27T21:35:35,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:35,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a1bb9b05a472888bd945369e4af06a:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:35:35,500 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:35:35,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:35,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a1bb9b05a472888bd945369e4af06a:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:35:35,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:35,500 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:35:35,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a1bb9b05a472888bd945369e4af06a:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:35:35,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:35:35,501 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:35:35,501 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:35:35,501 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): d9a1bb9b05a472888bd945369e4af06a/B is initiating minor compaction (all files) 2024-11-27T21:35:35,501 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): d9a1bb9b05a472888bd945369e4af06a/A is initiating minor compaction (all files) 2024-11-27T21:35:35,501 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a1bb9b05a472888bd945369e4af06a/B in TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:35,501 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a1bb9b05a472888bd945369e4af06a/A in TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:35,501 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/2aa5d24831ab421397dbcbe8cfa4a350, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/7482c3589fb54e5ea61626332319485a, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/c576f6e080094a80ba1a5384c4948823] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp, totalSize=35.9 K 2024-11-27T21:35:35,501 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/50e03acc026743a092d7d87aacfe80e6, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/b7b34394407a4fcb92018f884903559a, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/a263947548b54002acecd65192ff7652] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp, totalSize=35.9 K 2024-11-27T21:35:35,501 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 2aa5d24831ab421397dbcbe8cfa4a350, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732743334105 2024-11-27T21:35:35,502 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 7482c3589fb54e5ea61626332319485a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732743334133 2024-11-27T21:35:35,503 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting c576f6e080094a80ba1a5384c4948823, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732743334786 2024-11-27T21:35:35,504 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50e03acc026743a092d7d87aacfe80e6, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732743334105 2024-11-27T21:35:35,505 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7b34394407a4fcb92018f884903559a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732743334133 2024-11-27T21:35:35,505 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting a263947548b54002acecd65192ff7652, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732743334786 2024-11-27T21:35:35,513 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a1bb9b05a472888bd945369e4af06a#B#compaction#244 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:35,514 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/fc15cb4cc6d648dcaf86e2b390796b4b is 50, key is test_row_0/B:col10/1732743334791/Put/seqid=0 2024-11-27T21:35:35,523 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a1bb9b05a472888bd945369e4af06a#A#compaction#245 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:35,523 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/1c31b61189994ad78e8b0f7ccdbd0ebc is 50, key is test_row_0/A:col10/1732743334791/Put/seqid=0 2024-11-27T21:35:35,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742113_1289 (size=12561) 2024-11-27T21:35:35,537 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/fc15cb4cc6d648dcaf86e2b390796b4b as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/fc15cb4cc6d648dcaf86e2b390796b4b 2024-11-27T21:35:35,545 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9a1bb9b05a472888bd945369e4af06a/B of d9a1bb9b05a472888bd945369e4af06a into fc15cb4cc6d648dcaf86e2b390796b4b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:35,545 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:35,545 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., storeName=d9a1bb9b05a472888bd945369e4af06a/B, priority=13, startTime=1732743335500; duration=0sec 2024-11-27T21:35:35,545 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:35:35,545 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a1bb9b05a472888bd945369e4af06a:B 2024-11-27T21:35:35,545 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:35:35,547 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:35:35,547 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): d9a1bb9b05a472888bd945369e4af06a/C is initiating minor compaction (all files) 2024-11-27T21:35:35,547 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a1bb9b05a472888bd945369e4af06a/C in TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:35,547 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/ad8d4284f72544eea73700368529be18, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/26ccb361a1d24b7ab82d5ddf52fb2e68, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/0513dd2a4cd140dc9a4b42e3472bf730] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp, totalSize=35.9 K 2024-11-27T21:35:35,548 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting ad8d4284f72544eea73700368529be18, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732743334105 2024-11-27T21:35:35,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742114_1290 (size=12561) 2024-11-27T21:35:35,548 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 26ccb361a1d24b7ab82d5ddf52fb2e68, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732743334133 2024-11-27T21:35:35,549 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 0513dd2a4cd140dc9a4b42e3472bf730, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732743334786 2024-11-27T21:35:35,554 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/1c31b61189994ad78e8b0f7ccdbd0ebc as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/1c31b61189994ad78e8b0f7ccdbd0ebc 2024-11-27T21:35:35,558 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a1bb9b05a472888bd945369e4af06a#C#compaction#246 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:35,559 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/8f0dc41f8d934f4da7213a51afe29f11 is 50, key is test_row_0/C:col10/1732743334791/Put/seqid=0 2024-11-27T21:35:35,563 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9a1bb9b05a472888bd945369e4af06a/A of d9a1bb9b05a472888bd945369e4af06a into 1c31b61189994ad78e8b0f7ccdbd0ebc(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:35,563 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:35,563 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., storeName=d9a1bb9b05a472888bd945369e4af06a/A, priority=13, startTime=1732743335500; duration=0sec 2024-11-27T21:35:35,563 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:35,563 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a1bb9b05a472888bd945369e4af06a:A 2024-11-27T21:35:35,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on d9a1bb9b05a472888bd945369e4af06a 2024-11-27T21:35:35,576 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:35:35,576 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d9a1bb9b05a472888bd945369e4af06a 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-27T21:35:35,581 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/dc99716cd61e4c5b89d65d43f7a4dd53 is 50, key is test_row_0/A:col10/1732743335460/Put/seqid=0 2024-11-27T21:35:35,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742115_1291 (size=12561) 2024-11-27T21:35:35,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-27T21:35:35,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742116_1292 (size=12151) 2024-11-27T21:35:35,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:35,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:35,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743395594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:35,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743395593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:35,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:35,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:35,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743395596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:35,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:35,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:35,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:35,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:35,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:35,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:35,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:35,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:35,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:35,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743395599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:35,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:35,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:35,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:35,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-27T21:35:35,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:35,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. as already flushing 2024-11-27T21:35:35,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:35,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:35,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:35,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:35,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:35,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743395703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:35,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:35,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:35,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:35,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:35,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743395703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:35,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:35,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:35,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:35,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:35,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743395703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:35,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:35,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:35,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:35,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:35,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743395713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:35,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:35,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:35,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:35,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-27T21:35:35,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:35,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. as already flushing 2024-11-27T21:35:35,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:35,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:35,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:35,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:35,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-27T21:35:35,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:35,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743395913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:35,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:35,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743395913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:35,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:35,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:35,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:35,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:35,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:35,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:35,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:35,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743395913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:35,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:35,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:35,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:35,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:35,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743395923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:35,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:35,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:35,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:35,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-27T21:35:35,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:35,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. as already flushing 2024-11-27T21:35:35,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:35,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:35,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:35,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:35,988 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/8f0dc41f8d934f4da7213a51afe29f11 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/8f0dc41f8d934f4da7213a51afe29f11 2024-11-27T21:35:35,993 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9a1bb9b05a472888bd945369e4af06a/C of d9a1bb9b05a472888bd945369e4af06a into 8f0dc41f8d934f4da7213a51afe29f11(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:35,993 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:35,993 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., storeName=d9a1bb9b05a472888bd945369e4af06a/C, priority=13, startTime=1732743335500; duration=0sec 2024-11-27T21:35:35,994 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:35,994 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a1bb9b05a472888bd945369e4af06a:C 2024-11-27T21:35:35,994 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/dc99716cd61e4c5b89d65d43f7a4dd53 2024-11-27T21:35:36,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/2708fd3769a24fbf84705fa79106dfaa is 50, key is test_row_0/B:col10/1732743335460/Put/seqid=0 2024-11-27T21:35:36,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742117_1293 (size=12151) 2024-11-27T21:35:36,007 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/2708fd3769a24fbf84705fa79106dfaa 2024-11-27T21:35:36,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/d5b4c85fdcca40f896c44891cb999a40 is 50, key is test_row_0/C:col10/1732743335460/Put/seqid=0 2024-11-27T21:35:36,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742118_1294 (size=12151) 2024-11-27T21:35:36,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/d5b4c85fdcca40f896c44891cb999a40 2024-11-27T21:35:36,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/dc99716cd61e4c5b89d65d43f7a4dd53 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/dc99716cd61e4c5b89d65d43f7a4dd53 2024-11-27T21:35:36,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/dc99716cd61e4c5b89d65d43f7a4dd53, entries=150, sequenceid=196, filesize=11.9 K 2024-11-27T21:35:36,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/2708fd3769a24fbf84705fa79106dfaa as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/2708fd3769a24fbf84705fa79106dfaa 2024-11-27T21:35:36,062 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/2708fd3769a24fbf84705fa79106dfaa, entries=150, sequenceid=196, filesize=11.9 K 2024-11-27T21:35:36,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/d5b4c85fdcca40f896c44891cb999a40 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/d5b4c85fdcca40f896c44891cb999a40 2024-11-27T21:35:36,071 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/d5b4c85fdcca40f896c44891cb999a40, entries=150, sequenceid=196, filesize=11.9 K 2024-11-27T21:35:36,072 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for d9a1bb9b05a472888bd945369e4af06a in 496ms, sequenceid=196, compaction requested=false 2024-11-27T21:35:36,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:36,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-27T21:35:36,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:36,091 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing d9a1bb9b05a472888bd945369e4af06a 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-27T21:35:36,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/f97ca61bb8794672a65b4ef7e8a2f002 is 50, key is test_row_0/A:col10/1732743335597/Put/seqid=0 2024-11-27T21:35:36,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-27T21:35:36,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742119_1295 (size=12151) 2024-11-27T21:35:36,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. as already flushing 2024-11-27T21:35:36,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on d9a1bb9b05a472888bd945369e4af06a 2024-11-27T21:35:36,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:36,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743396253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:36,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:36,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:36,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:36,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:36,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743396258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:36,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:36,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743396260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:36,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:36,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:36,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:36,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:36,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:36,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:36,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:36,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743396260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:36,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:36,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:36,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:36,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:36,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743396363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:36,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:36,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:36,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:36,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:36,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:36,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743396374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:36,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743396374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:36,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:36,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:36,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:36,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743396374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:36,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:36,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:36,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:36,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:36,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:36,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:36,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:36,519 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/f97ca61bb8794672a65b4ef7e8a2f002 2024-11-27T21:35:36,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/0a6d639af7da4edf836fb66a9e0bbf82 is 50, key is test_row_0/B:col10/1732743335597/Put/seqid=0 2024-11-27T21:35:36,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742120_1296 (size=12151) 2024-11-27T21:35:36,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:36,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743396573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:36,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:36,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:36,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:36,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:36,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743396584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:36,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:36,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:36,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743396584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:36,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:36,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:36,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:36,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:36,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:36,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:36,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743396584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:36,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:36,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:36,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:36,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-27T21:35:36,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:36,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743396883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:36,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:36,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:36,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:36,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:36,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743396893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:36,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:36,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:36,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:36,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:36,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743396893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:36,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:36,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:36,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:36,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:36,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743396894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:36,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:36,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:36,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:36,931 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/0a6d639af7da4edf836fb66a9e0bbf82 2024-11-27T21:35:36,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/1289e38d20d64b6fb4c7085a98f0397f is 50, key is test_row_0/C:col10/1732743335597/Put/seqid=0 2024-11-27T21:35:36,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742121_1297 (size=12151) 2024-11-27T21:35:36,943 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/1289e38d20d64b6fb4c7085a98f0397f 2024-11-27T21:35:36,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/f97ca61bb8794672a65b4ef7e8a2f002 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/f97ca61bb8794672a65b4ef7e8a2f002 2024-11-27T21:35:36,953 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/f97ca61bb8794672a65b4ef7e8a2f002, entries=150, sequenceid=213, filesize=11.9 K 2024-11-27T21:35:36,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/0a6d639af7da4edf836fb66a9e0bbf82 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/0a6d639af7da4edf836fb66a9e0bbf82 2024-11-27T21:35:36,958 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/0a6d639af7da4edf836fb66a9e0bbf82, entries=150, sequenceid=213, filesize=11.9 K 2024-11-27T21:35:36,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/1289e38d20d64b6fb4c7085a98f0397f as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/1289e38d20d64b6fb4c7085a98f0397f 2024-11-27T21:35:36,962 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/1289e38d20d64b6fb4c7085a98f0397f, entries=150, sequenceid=213, filesize=11.9 K 2024-11-27T21:35:36,963 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for d9a1bb9b05a472888bd945369e4af06a in 872ms, sequenceid=213, compaction requested=true 2024-11-27T21:35:36,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:36,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:36,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-27T21:35:36,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-27T21:35:36,966 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-27T21:35:36,966 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4860 sec 2024-11-27T21:35:36,968 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.4920 sec 2024-11-27T21:35:37,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on d9a1bb9b05a472888bd945369e4af06a 2024-11-27T21:35:37,394 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:35:37,394 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d9a1bb9b05a472888bd945369e4af06a 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-27T21:35:37,400 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/5d3eeb9f8aea4724a7386a4f7d75964d is 50, key is test_row_0/A:col10/1732743337393/Put/seqid=0 2024-11-27T21:35:37,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742122_1298 (size=12151) 2024-11-27T21:35:37,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:37,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743397414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:37,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:37,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:37,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:37,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:37,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743397417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:37,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:37,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:37,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:37,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:37,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743397418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:37,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:37,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:37,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:37,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:37,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743397418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:37,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:37,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:37,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:37,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:37,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743397523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:37,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:37,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:37,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:37,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:37,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743397524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:37,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:37,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:37,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:37,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:37,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743397534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:37,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:37,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:37,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:37,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:37,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743397534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:37,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:37,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:37,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:37,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-27T21:35:37,624 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:35:37,624 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:35:37,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-27T21:35:37,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-27T21:35:37,627 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:35:37,627 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:35:37,628 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:35:37,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-27T21:35:37,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:37,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743397733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:37,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:37,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:37,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:37,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743397734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:37,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:37,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:37,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:37,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:37,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:37,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743397743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:37,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:37,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:37,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:37,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:37,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743397743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:37,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:37,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:37,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:37,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-27T21:35:37,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:37,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. as already flushing 2024-11-27T21:35:37,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:37,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:37,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:37,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:37,805 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/5d3eeb9f8aea4724a7386a4f7d75964d 2024-11-27T21:35:37,813 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/b435466a7b94451686b6cb07af2f223a is 50, key is test_row_0/B:col10/1732743337393/Put/seqid=0 2024-11-27T21:35:37,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742123_1299 (size=12151) 2024-11-27T21:35:37,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-27T21:35:37,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:37,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. as already flushing 2024-11-27T21:35:37,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:37,934 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:37,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:37,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:37,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-27T21:35:38,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:38,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743398044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:38,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:38,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:38,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:38,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743398044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:38,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:38,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:38,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:38,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:38,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:38,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743398053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:38,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:38,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:38,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:38,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:38,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743398053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:38,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:38,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:38,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:38,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-27T21:35:38,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:38,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. as already flushing 2024-11-27T21:35:38,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:38,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:38,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:38,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:38,217 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/b435466a7b94451686b6cb07af2f223a 2024-11-27T21:35:38,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/74ef579ef94147b484428e69f3733619 is 50, key is test_row_0/C:col10/1732743337393/Put/seqid=0 2024-11-27T21:35:38,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742124_1300 (size=12151) 2024-11-27T21:35:38,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/74ef579ef94147b484428e69f3733619 2024-11-27T21:35:38,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/5d3eeb9f8aea4724a7386a4f7d75964d as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/5d3eeb9f8aea4724a7386a4f7d75964d 2024-11-27T21:35:38,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/5d3eeb9f8aea4724a7386a4f7d75964d, entries=150, sequenceid=234, filesize=11.9 K 2024-11-27T21:35:38,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/b435466a7b94451686b6cb07af2f223a as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/b435466a7b94451686b6cb07af2f223a 2024-11-27T21:35:38,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-27T21:35:38,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:38,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. as already flushing 2024-11-27T21:35:38,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:38,240 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:38,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:38,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:38,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/b435466a7b94451686b6cb07af2f223a, entries=150, sequenceid=234, filesize=11.9 K 2024-11-27T21:35:38,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/74ef579ef94147b484428e69f3733619 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/74ef579ef94147b484428e69f3733619 2024-11-27T21:35:38,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/74ef579ef94147b484428e69f3733619, entries=150, sequenceid=234, filesize=11.9 K 2024-11-27T21:35:38,250 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for d9a1bb9b05a472888bd945369e4af06a in 856ms, sequenceid=234, compaction requested=true 2024-11-27T21:35:38,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:38,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a1bb9b05a472888bd945369e4af06a:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:35:38,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:38,251 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:35:38,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a1bb9b05a472888bd945369e4af06a:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:35:38,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:38,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a1bb9b05a472888bd945369e4af06a:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:35:38,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:35:38,251 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:35:38,252 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:35:38,252 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:35:38,252 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): d9a1bb9b05a472888bd945369e4af06a/B is initiating minor compaction (all files) 2024-11-27T21:35:38,252 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): d9a1bb9b05a472888bd945369e4af06a/A is initiating minor compaction (all files) 2024-11-27T21:35:38,252 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a1bb9b05a472888bd945369e4af06a/A in TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:38,252 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a1bb9b05a472888bd945369e4af06a/B in TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:38,252 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/1c31b61189994ad78e8b0f7ccdbd0ebc, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/dc99716cd61e4c5b89d65d43f7a4dd53, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/f97ca61bb8794672a65b4ef7e8a2f002, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/5d3eeb9f8aea4724a7386a4f7d75964d] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp, totalSize=47.9 K 2024-11-27T21:35:38,252 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/fc15cb4cc6d648dcaf86e2b390796b4b, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/2708fd3769a24fbf84705fa79106dfaa, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/0a6d639af7da4edf836fb66a9e0bbf82, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/b435466a7b94451686b6cb07af2f223a] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp, totalSize=47.9 K 2024-11-27T21:35:38,252 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c31b61189994ad78e8b0f7ccdbd0ebc, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732743334786 2024-11-27T21:35:38,252 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting fc15cb4cc6d648dcaf86e2b390796b4b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732743334786 2024-11-27T21:35:38,253 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc99716cd61e4c5b89d65d43f7a4dd53, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732743335460 2024-11-27T21:35:38,253 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 2708fd3769a24fbf84705fa79106dfaa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732743335460 2024-11-27T21:35:38,253 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting f97ca61bb8794672a65b4ef7e8a2f002, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732743335589 2024-11-27T21:35:38,253 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a6d639af7da4edf836fb66a9e0bbf82, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732743335589 2024-11-27T21:35:38,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-27T21:35:38,254 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting b435466a7b94451686b6cb07af2f223a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732743336249 2024-11-27T21:35:38,254 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d3eeb9f8aea4724a7386a4f7d75964d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732743336249 2024-11-27T21:35:38,262 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a1bb9b05a472888bd945369e4af06a#B#compaction#256 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:38,262 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/a1655af8e8694749be447ade6df04777 is 50, key is test_row_0/B:col10/1732743337393/Put/seqid=0 2024-11-27T21:35:38,262 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a1bb9b05a472888bd945369e4af06a#A#compaction#257 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:38,263 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/71022d59ede146098e5799bd0d5ad945 is 50, key is test_row_0/A:col10/1732743337393/Put/seqid=0 2024-11-27T21:35:38,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742125_1301 (size=12697) 2024-11-27T21:35:38,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742126_1302 (size=12697) 2024-11-27T21:35:38,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-27T21:35:38,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:38,393 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing d9a1bb9b05a472888bd945369e4af06a 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-27T21:35:38,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/454cc23617e0415a99d8127435b6fde9 is 50, key is test_row_0/A:col10/1732743337417/Put/seqid=0 2024-11-27T21:35:38,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742127_1303 (size=12151) 2024-11-27T21:35:38,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on d9a1bb9b05a472888bd945369e4af06a 2024-11-27T21:35:38,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. as already flushing 2024-11-27T21:35:38,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:38,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743398581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:38,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:38,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:38,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:38,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:38,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743398582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:38,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:38,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:38,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:38,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:38,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743398582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:38,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:38,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:38,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:38,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:38,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743398583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:38,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:38,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:38,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:38,679 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/a1655af8e8694749be447ade6df04777 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/a1655af8e8694749be447ade6df04777 2024-11-27T21:35:38,682 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/71022d59ede146098e5799bd0d5ad945 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/71022d59ede146098e5799bd0d5ad945 2024-11-27T21:35:38,684 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d9a1bb9b05a472888bd945369e4af06a/B of d9a1bb9b05a472888bd945369e4af06a into a1655af8e8694749be447ade6df04777(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:38,684 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:38,684 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., storeName=d9a1bb9b05a472888bd945369e4af06a/B, priority=12, startTime=1732743338251; duration=0sec 2024-11-27T21:35:38,684 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:35:38,684 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a1bb9b05a472888bd945369e4af06a:B 2024-11-27T21:35:38,684 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:35:38,686 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:35:38,686 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): d9a1bb9b05a472888bd945369e4af06a/C is initiating minor compaction (all files) 2024-11-27T21:35:38,686 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a1bb9b05a472888bd945369e4af06a/C in TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:38,686 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/8f0dc41f8d934f4da7213a51afe29f11, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/d5b4c85fdcca40f896c44891cb999a40, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/1289e38d20d64b6fb4c7085a98f0397f, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/74ef579ef94147b484428e69f3733619] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp, totalSize=47.9 K 2024-11-27T21:35:38,686 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f0dc41f8d934f4da7213a51afe29f11, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732743334786 2024-11-27T21:35:38,687 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting d5b4c85fdcca40f896c44891cb999a40, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732743335460 2024-11-27T21:35:38,687 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 1289e38d20d64b6fb4c7085a98f0397f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732743335589 2024-11-27T21:35:38,687 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d9a1bb9b05a472888bd945369e4af06a/A of d9a1bb9b05a472888bd945369e4af06a into 71022d59ede146098e5799bd0d5ad945(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:38,687 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:38,687 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., storeName=d9a1bb9b05a472888bd945369e4af06a/A, priority=12, startTime=1732743338250; duration=0sec 2024-11-27T21:35:38,687 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 74ef579ef94147b484428e69f3733619, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732743336249 2024-11-27T21:35:38,687 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:38,688 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a1bb9b05a472888bd945369e4af06a:A 2024-11-27T21:35:38,695 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a1bb9b05a472888bd945369e4af06a#C#compaction#259 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:38,695 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/0cdd4cd911504da7a88dc99c44e2194a is 50, key is test_row_0/C:col10/1732743337393/Put/seqid=0 2024-11-27T21:35:38,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:38,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743398694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:38,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:38,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:38,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:38,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743398694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:38,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:38,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:38,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:38,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:38,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:38,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743398694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:38,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:38,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:38,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:38,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:38,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743398694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:38,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:38,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:38,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:38,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742128_1304 (size=12697) 2024-11-27T21:35:38,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-27T21:35:38,803 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/454cc23617e0415a99d8127435b6fde9 2024-11-27T21:35:38,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/0b164efbef54486e892435e497a13a79 is 50, key is test_row_0/B:col10/1732743337417/Put/seqid=0 2024-11-27T21:35:38,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742129_1305 (size=12151) 2024-11-27T21:35:38,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:38,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743398904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:38,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:38,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:38,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:38,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:38,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743398904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:38,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:38,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:38,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:38,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:38,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743398904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:38,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:38,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743398904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:38,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:38,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:38,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:38,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:38,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:38,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:39,104 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/0cdd4cd911504da7a88dc99c44e2194a as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/0cdd4cd911504da7a88dc99c44e2194a 2024-11-27T21:35:39,109 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d9a1bb9b05a472888bd945369e4af06a/C of d9a1bb9b05a472888bd945369e4af06a into 0cdd4cd911504da7a88dc99c44e2194a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:39,109 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:39,109 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., storeName=d9a1bb9b05a472888bd945369e4af06a/C, priority=12, startTime=1732743338251; duration=0sec 2024-11-27T21:35:39,109 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:39,109 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a1bb9b05a472888bd945369e4af06a:C 2024-11-27T21:35:39,215 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/0b164efbef54486e892435e497a13a79 2024-11-27T21:35:39,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:39,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743399213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:39,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:39,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:39,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:39,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:39,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743399213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:39,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:39,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:39,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:39,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:39,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743399213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:39,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:39,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:39,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:39,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:39,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743399214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:39,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:39,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:39,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:39,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/dad9770f1db04e6a8ff54634185244de is 50, key is test_row_0/C:col10/1732743337417/Put/seqid=0 2024-11-27T21:35:39,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742130_1306 (size=12151) 2024-11-27T21:35:39,626 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/dad9770f1db04e6a8ff54634185244de 2024-11-27T21:35:39,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/454cc23617e0415a99d8127435b6fde9 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/454cc23617e0415a99d8127435b6fde9 2024-11-27T21:35:39,635 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/454cc23617e0415a99d8127435b6fde9, entries=150, sequenceid=250, filesize=11.9 K 2024-11-27T21:35:39,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/0b164efbef54486e892435e497a13a79 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/0b164efbef54486e892435e497a13a79 2024-11-27T21:35:39,639 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/0b164efbef54486e892435e497a13a79, entries=150, sequenceid=250, filesize=11.9 K 2024-11-27T21:35:39,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/dad9770f1db04e6a8ff54634185244de as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/dad9770f1db04e6a8ff54634185244de 2024-11-27T21:35:39,644 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/dad9770f1db04e6a8ff54634185244de, entries=150, sequenceid=250, filesize=11.9 K 2024-11-27T21:35:39,645 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for d9a1bb9b05a472888bd945369e4af06a in 1252ms, sequenceid=250, compaction requested=false 2024-11-27T21:35:39,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:39,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:39,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-27T21:35:39,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-27T21:35:39,649 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-27T21:35:39,649 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0190 sec 2024-11-27T21:35:39,652 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 2.0260 sec 2024-11-27T21:35:39,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on d9a1bb9b05a472888bd945369e4af06a 2024-11-27T21:35:39,726 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:35:39,726 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d9a1bb9b05a472888bd945369e4af06a 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-27T21:35:39,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/d9d0dde632974ee89ef38729c9a9aae1 is 50, key is test_row_0/A:col10/1732743338579/Put/seqid=0 2024-11-27T21:35:39,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742131_1307 (size=14741) 2024-11-27T21:35:39,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:39,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743399737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:39,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:39,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:39,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:39,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:39,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743399738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:39,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:39,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:39,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:39,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:39,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743399739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:39,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:39,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:39,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:39,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:39,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743399740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:39,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:39,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:39,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:39,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-27T21:35:39,773 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:35:39,774 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:35:39,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-27T21:35:39,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-27T21:35:39,775 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:35:39,776 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:35:39,776 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:35:39,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:39,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743399843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:39,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:39,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:39,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:39,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:39,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743399853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:39,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:39,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:39,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:39,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:39,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743399853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:39,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:39,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:39,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743399853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:39,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:39,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:39,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:39,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:39,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:39,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-27T21:35:39,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-27T21:35:39,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:39,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. as already flushing 2024-11-27T21:35:39,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:39,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:39,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:39,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:40,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:40,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743400053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:40,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:40,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:40,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:40,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:40,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743400063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:40,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743400063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:40,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:40,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743400063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:40,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:40,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:40,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:40,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:40,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:40,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:40,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:40,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:40,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-27T21:35:40,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:40,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. as already flushing 2024-11-27T21:35:40,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:40,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:40,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-27T21:35:40,137 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/d9d0dde632974ee89ef38729c9a9aae1 2024-11-27T21:35:40,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/f43dc7d8eaca45ab8282b536f478ea55 is 50, key is test_row_0/B:col10/1732743338579/Put/seqid=0 2024-11-27T21:35:40,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742132_1308 (size=12301) 2024-11-27T21:35:40,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-27T21:35:40,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:40,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. as already flushing 2024-11-27T21:35:40,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:40,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:40,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:40,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:40,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:40,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743400364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:40,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:40,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:40,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:40,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:40,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743400373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:40,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:40,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:40,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:40,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:40,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743400373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:40,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:40,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:40,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:40,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743400373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:40,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:40,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:40,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:40,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:40,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-27T21:35:40,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:40,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. as already flushing 2024-11-27T21:35:40,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:40,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:40,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:40,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:40,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-27T21:35:40,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-27T21:35:40,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:40,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. as already flushing 2024-11-27T21:35:40,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:40,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:40,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:40,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:40,548 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/f43dc7d8eaca45ab8282b536f478ea55 2024-11-27T21:35:40,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/990241e5cd1d4507b13ed61cb9e01823 is 50, key is test_row_0/C:col10/1732743338579/Put/seqid=0 2024-11-27T21:35:40,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742133_1309 (size=12301) 2024-11-27T21:35:40,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-27T21:35:40,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:40,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. as already flushing 2024-11-27T21:35:40,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:40,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:40,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:40,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:40,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-27T21:35:40,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:40,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. as already flushing 2024-11-27T21:35:40,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:40,847 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:40,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:40,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:40,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:40,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743400873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:40,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:40,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:40,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:40,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:40,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743400883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:40,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:40,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743400883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:40,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:40,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:40,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:40,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743400883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:40,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:40,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:40,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:40,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:40,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:40,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:40,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:40,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-27T21:35:40,960 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/990241e5cd1d4507b13ed61cb9e01823 2024-11-27T21:35:40,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/d9d0dde632974ee89ef38729c9a9aae1 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/d9d0dde632974ee89ef38729c9a9aae1 2024-11-27T21:35:40,969 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/d9d0dde632974ee89ef38729c9a9aae1, entries=200, sequenceid=276, filesize=14.4 K 2024-11-27T21:35:40,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/f43dc7d8eaca45ab8282b536f478ea55 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/f43dc7d8eaca45ab8282b536f478ea55 2024-11-27T21:35:40,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/f43dc7d8eaca45ab8282b536f478ea55, entries=150, sequenceid=276, filesize=12.0 K 2024-11-27T21:35:40,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/990241e5cd1d4507b13ed61cb9e01823 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/990241e5cd1d4507b13ed61cb9e01823 2024-11-27T21:35:40,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/990241e5cd1d4507b13ed61cb9e01823, entries=150, sequenceid=276, filesize=12.0 K 2024-11-27T21:35:40,978 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for d9a1bb9b05a472888bd945369e4af06a in 1252ms, sequenceid=276, compaction requested=true 2024-11-27T21:35:40,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:40,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a1bb9b05a472888bd945369e4af06a:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:35:40,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:40,978 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:35:40,978 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:35:40,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a1bb9b05a472888bd945369e4af06a:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:35:40,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:40,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a1bb9b05a472888bd945369e4af06a:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:35:40,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:35:40,979 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:35:40,979 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39589 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:35:40,979 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): d9a1bb9b05a472888bd945369e4af06a/A is initiating minor compaction (all files) 2024-11-27T21:35:40,979 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): d9a1bb9b05a472888bd945369e4af06a/B is initiating minor compaction (all files) 2024-11-27T21:35:40,979 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a1bb9b05a472888bd945369e4af06a/A in TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:40,979 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a1bb9b05a472888bd945369e4af06a/B in TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:40,979 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/a1655af8e8694749be447ade6df04777, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/0b164efbef54486e892435e497a13a79, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/f43dc7d8eaca45ab8282b536f478ea55] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp, totalSize=36.3 K 2024-11-27T21:35:40,979 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/71022d59ede146098e5799bd0d5ad945, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/454cc23617e0415a99d8127435b6fde9, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/d9d0dde632974ee89ef38729c9a9aae1] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp, totalSize=38.7 K 2024-11-27T21:35:40,980 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting a1655af8e8694749be447ade6df04777, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732743336249 2024-11-27T21:35:40,980 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71022d59ede146098e5799bd0d5ad945, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732743336249 2024-11-27T21:35:40,980 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b164efbef54486e892435e497a13a79, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732743337412 2024-11-27T21:35:40,980 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 454cc23617e0415a99d8127435b6fde9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732743337412 2024-11-27T21:35:40,981 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting f43dc7d8eaca45ab8282b536f478ea55, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732743338579 2024-11-27T21:35:40,981 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9d0dde632974ee89ef38729c9a9aae1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732743338579 2024-11-27T21:35:40,988 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a1bb9b05a472888bd945369e4af06a#B#compaction#265 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:40,988 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a1bb9b05a472888bd945369e4af06a#A#compaction#266 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:40,989 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/b8a5624851ca4e9081379ba2c462e592 is 50, key is test_row_0/B:col10/1732743338579/Put/seqid=0 2024-11-27T21:35:40,989 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/2de6feb3121f48578aa79c84db65e2e6 is 50, key is test_row_0/A:col10/1732743338579/Put/seqid=0 2024-11-27T21:35:41,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-27T21:35:41,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:41,000 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing d9a1bb9b05a472888bd945369e4af06a 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-27T21:35:41,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/171b6ffda9fd4634a8b7c2c6bd7a9508 is 50, key is test_row_0/A:col10/1732743339738/Put/seqid=0 2024-11-27T21:35:41,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742134_1310 (size=12949) 2024-11-27T21:35:41,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742136_1312 (size=12301) 2024-11-27T21:35:41,023 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/171b6ffda9fd4634a8b7c2c6bd7a9508 2024-11-27T21:35:41,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742135_1311 (size=12949) 2024-11-27T21:35:41,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/2c7a0b13b0fb453b829101525a7155c8 is 50, key is test_row_0/B:col10/1732743339738/Put/seqid=0 2024-11-27T21:35:41,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742137_1313 (size=12301) 2024-11-27T21:35:41,410 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/2de6feb3121f48578aa79c84db65e2e6 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/2de6feb3121f48578aa79c84db65e2e6 2024-11-27T21:35:41,415 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9a1bb9b05a472888bd945369e4af06a/A of d9a1bb9b05a472888bd945369e4af06a into 2de6feb3121f48578aa79c84db65e2e6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:41,415 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:41,415 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., storeName=d9a1bb9b05a472888bd945369e4af06a/A, priority=13, startTime=1732743340978; duration=0sec 2024-11-27T21:35:41,415 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:35:41,415 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a1bb9b05a472888bd945369e4af06a:A 2024-11-27T21:35:41,415 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:35:41,417 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:35:41,417 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): d9a1bb9b05a472888bd945369e4af06a/C is initiating minor compaction (all files) 2024-11-27T21:35:41,417 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a1bb9b05a472888bd945369e4af06a/C in TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:41,417 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/0cdd4cd911504da7a88dc99c44e2194a, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/dad9770f1db04e6a8ff54634185244de, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/990241e5cd1d4507b13ed61cb9e01823] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp, totalSize=36.3 K 2024-11-27T21:35:41,417 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0cdd4cd911504da7a88dc99c44e2194a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732743336249 2024-11-27T21:35:41,418 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting dad9770f1db04e6a8ff54634185244de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732743337412 2024-11-27T21:35:41,418 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 990241e5cd1d4507b13ed61cb9e01823, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732743338579 2024-11-27T21:35:41,425 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a1bb9b05a472888bd945369e4af06a#C#compaction#269 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:41,425 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/7a11115efbe24efe88f29e54eaaca27b is 50, key is test_row_0/C:col10/1732743338579/Put/seqid=0 2024-11-27T21:35:41,427 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/b8a5624851ca4e9081379ba2c462e592 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/b8a5624851ca4e9081379ba2c462e592 2024-11-27T21:35:41,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742138_1314 (size=12949) 2024-11-27T21:35:41,434 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9a1bb9b05a472888bd945369e4af06a/B of d9a1bb9b05a472888bd945369e4af06a into b8a5624851ca4e9081379ba2c462e592(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:41,434 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:41,434 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., storeName=d9a1bb9b05a472888bd945369e4af06a/B, priority=13, startTime=1732743340978; duration=0sec 2024-11-27T21:35:41,434 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:41,434 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a1bb9b05a472888bd945369e4af06a:B 2024-11-27T21:35:41,435 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/2c7a0b13b0fb453b829101525a7155c8 2024-11-27T21:35:41,436 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/7a11115efbe24efe88f29e54eaaca27b as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/7a11115efbe24efe88f29e54eaaca27b 2024-11-27T21:35:41,443 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9a1bb9b05a472888bd945369e4af06a/C of d9a1bb9b05a472888bd945369e4af06a into 7a11115efbe24efe88f29e54eaaca27b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:41,443 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:41,443 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., storeName=d9a1bb9b05a472888bd945369e4af06a/C, priority=13, startTime=1732743340979; duration=0sec 2024-11-27T21:35:41,443 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:41,443 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a1bb9b05a472888bd945369e4af06a:C 2024-11-27T21:35:41,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/e427ef3b0ca54ce19369371ebd81a1b9 is 50, key is test_row_0/C:col10/1732743339738/Put/seqid=0 2024-11-27T21:35:41,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742139_1315 (size=12301) 2024-11-27T21:35:41,850 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/e427ef3b0ca54ce19369371ebd81a1b9 2024-11-27T21:35:41,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/171b6ffda9fd4634a8b7c2c6bd7a9508 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/171b6ffda9fd4634a8b7c2c6bd7a9508 2024-11-27T21:35:41,859 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/171b6ffda9fd4634a8b7c2c6bd7a9508, entries=150, sequenceid=289, filesize=12.0 K 2024-11-27T21:35:41,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/2c7a0b13b0fb453b829101525a7155c8 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/2c7a0b13b0fb453b829101525a7155c8 2024-11-27T21:35:41,865 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/2c7a0b13b0fb453b829101525a7155c8, entries=150, sequenceid=289, filesize=12.0 K 2024-11-27T21:35:41,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/e427ef3b0ca54ce19369371ebd81a1b9 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/e427ef3b0ca54ce19369371ebd81a1b9 2024-11-27T21:35:41,870 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/e427ef3b0ca54ce19369371ebd81a1b9, entries=150, sequenceid=289, filesize=12.0 K 2024-11-27T21:35:41,871 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=0 B/0 for d9a1bb9b05a472888bd945369e4af06a in 871ms, sequenceid=289, compaction requested=false 2024-11-27T21:35:41,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:41,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:41,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-27T21:35:41,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-27T21:35:41,874 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-27T21:35:41,875 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0960 sec 2024-11-27T21:35:41,877 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 2.1010 sec 2024-11-27T21:35:41,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on d9a1bb9b05a472888bd945369e4af06a 2024-11-27T21:35:41,902 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:35:41,903 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d9a1bb9b05a472888bd945369e4af06a 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T21:35:41,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/445828923fe94b69a5958ed88e894924 is 50, key is test_row_0/A:col10/1732743341896/Put/seqid=0 2024-11-27T21:35:41,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742140_1316 (size=14741) 2024-11-27T21:35:41,915 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/445828923fe94b69a5958ed88e894924 2024-11-27T21:35:41,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/557bbc93ca904e8c8794533ba171392d is 50, key is test_row_0/B:col10/1732743341896/Put/seqid=0 2024-11-27T21:35:41,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742141_1317 (size=12301) 2024-11-27T21:35:41,930 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/557bbc93ca904e8c8794533ba171392d 2024-11-27T21:35:41,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-27T21:35:41,934 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:35:41,934 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:35:41,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees 2024-11-27T21:35:41,936 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:35:41,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-27T21:35:41,937 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:35:41,937 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:35:41,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/ef4aef0851454354a71f0b920672649f is 50, key is test_row_0/C:col10/1732743341896/Put/seqid=0 2024-11-27T21:35:41,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:41,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743401933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:41,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:41,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743401934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:41,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:41,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:41,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:41,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:41,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743401934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:41,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:41,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:41,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:41,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:41,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:41,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:41,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:41,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743401939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:41,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:41,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:41,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:41,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742142_1318 (size=12301) 2024-11-27T21:35:42,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-27T21:35:42,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:42,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743402043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:42,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:42,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:42,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:42,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:42,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743402044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:42,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:42,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:42,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:42,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:42,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743402044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:42,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:42,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:42,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:42,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:42,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743402053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:42,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:42,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:42,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:42,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-27T21:35:42,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:42,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. as already flushing 2024-11-27T21:35:42,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:42,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:42,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:42,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:42,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-27T21:35:42,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:42,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. as already flushing 2024-11-27T21:35:42,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:42,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:42,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:42,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:42,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-27T21:35:42,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:42,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743402254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:42,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:42,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:42,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743402254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:42,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:42,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:42,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:42,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:42,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:42,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:42,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743402254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:42,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:42,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:42,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:42,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:42,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743402263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:42,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:42,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:42,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:42,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/ef4aef0851454354a71f0b920672649f 2024-11-27T21:35:42,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/445828923fe94b69a5958ed88e894924 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/445828923fe94b69a5958ed88e894924 2024-11-27T21:35:42,361 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/445828923fe94b69a5958ed88e894924, entries=200, sequenceid=305, filesize=14.4 K 2024-11-27T21:35:42,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/557bbc93ca904e8c8794533ba171392d as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/557bbc93ca904e8c8794533ba171392d 2024-11-27T21:35:42,365 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/557bbc93ca904e8c8794533ba171392d, entries=150, sequenceid=305, filesize=12.0 K 2024-11-27T21:35:42,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/ef4aef0851454354a71f0b920672649f as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/ef4aef0851454354a71f0b920672649f 2024-11-27T21:35:42,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/ef4aef0851454354a71f0b920672649f, entries=150, sequenceid=305, filesize=12.0 K 2024-11-27T21:35:42,370 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for d9a1bb9b05a472888bd945369e4af06a in 468ms, sequenceid=305, compaction requested=true 2024-11-27T21:35:42,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:42,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a1bb9b05a472888bd945369e4af06a:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:35:42,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:42,371 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:35:42,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a1bb9b05a472888bd945369e4af06a:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:35:42,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:42,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a1bb9b05a472888bd945369e4af06a:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:35:42,371 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:35:42,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:35:42,372 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:35:42,372 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:35:42,372 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): d9a1bb9b05a472888bd945369e4af06a/B is initiating minor compaction (all files) 2024-11-27T21:35:42,372 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): d9a1bb9b05a472888bd945369e4af06a/A is initiating minor compaction (all files) 2024-11-27T21:35:42,372 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a1bb9b05a472888bd945369e4af06a/B in TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:42,372 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a1bb9b05a472888bd945369e4af06a/A in TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:42,372 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/b8a5624851ca4e9081379ba2c462e592, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/2c7a0b13b0fb453b829101525a7155c8, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/557bbc93ca904e8c8794533ba171392d] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp, totalSize=36.7 K 2024-11-27T21:35:42,372 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/2de6feb3121f48578aa79c84db65e2e6, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/171b6ffda9fd4634a8b7c2c6bd7a9508, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/445828923fe94b69a5958ed88e894924] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp, totalSize=39.1 K 2024-11-27T21:35:42,372 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting b8a5624851ca4e9081379ba2c462e592, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732743338579 2024-11-27T21:35:42,373 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2de6feb3121f48578aa79c84db65e2e6, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732743338579 2024-11-27T21:35:42,373 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c7a0b13b0fb453b829101525a7155c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732743339732 2024-11-27T21:35:42,373 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 171b6ffda9fd4634a8b7c2c6bd7a9508, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732743339732 2024-11-27T21:35:42,373 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 557bbc93ca904e8c8794533ba171392d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1732743341894 2024-11-27T21:35:42,373 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 445828923fe94b69a5958ed88e894924, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1732743341894 2024-11-27T21:35:42,381 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a1bb9b05a472888bd945369e4af06a#B#compaction#274 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:42,382 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/563173fd116e474bb5a1e485f5248618 is 50, key is test_row_0/B:col10/1732743341896/Put/seqid=0 2024-11-27T21:35:42,384 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a1bb9b05a472888bd945369e4af06a#A#compaction#275 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:42,385 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/59bd176eded74c8e941c9065850da4b1 is 50, key is test_row_0/A:col10/1732743341896/Put/seqid=0 2024-11-27T21:35:42,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-27T21:35:42,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:42,395 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2902): Flushing d9a1bb9b05a472888bd945369e4af06a 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-27T21:35:42,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742144_1320 (size=13051) 2024-11-27T21:35:42,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742143_1319 (size=13051) 2024-11-27T21:35:42,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/8020c759942a47e0a347f15ff5555431 is 50, key is test_row_0/A:col10/1732743341932/Put/seqid=0 2024-11-27T21:35:42,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742145_1321 (size=12301) 2024-11-27T21:35:42,431 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/8020c759942a47e0a347f15ff5555431 2024-11-27T21:35:42,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/57e11db7a2eb49db98ff58abd49b3604 is 50, key is test_row_0/B:col10/1732743341932/Put/seqid=0 2024-11-27T21:35:42,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742146_1322 (size=12301) 2024-11-27T21:35:42,480 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/57e11db7a2eb49db98ff58abd49b3604 2024-11-27T21:35:42,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/16a358771b7847eba3ff9b4dd5c413c1 is 50, key is test_row_0/C:col10/1732743341932/Put/seqid=0 2024-11-27T21:35:42,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742147_1323 (size=12301) 2024-11-27T21:35:42,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-27T21:35:42,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on d9a1bb9b05a472888bd945369e4af06a 2024-11-27T21:35:42,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. as already flushing 2024-11-27T21:35:42,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:42,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743402586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:42,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:42,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:42,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743402607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:42,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:42,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:42,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:42,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:42,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:42,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:42,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743402607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:42,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:42,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:42,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:42,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743402607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:42,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:42,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:42,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:42,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:42,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:42,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743402713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:42,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:42,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:42,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:42,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:42,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743402723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:42,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:42,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:42,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743402724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:42,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:42,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:42,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:42,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743402724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:42,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:42,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:42,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:42,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:42,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:42,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:42,815 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/563173fd116e474bb5a1e485f5248618 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/563173fd116e474bb5a1e485f5248618 2024-11-27T21:35:42,816 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/59bd176eded74c8e941c9065850da4b1 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/59bd176eded74c8e941c9065850da4b1 2024-11-27T21:35:42,821 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9a1bb9b05a472888bd945369e4af06a/B of d9a1bb9b05a472888bd945369e4af06a into 563173fd116e474bb5a1e485f5248618(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:42,822 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:42,822 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., storeName=d9a1bb9b05a472888bd945369e4af06a/B, priority=13, startTime=1732743342371; duration=0sec 2024-11-27T21:35:42,822 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:35:42,822 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a1bb9b05a472888bd945369e4af06a:B 2024-11-27T21:35:42,822 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:35:42,822 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9a1bb9b05a472888bd945369e4af06a/A of d9a1bb9b05a472888bd945369e4af06a into 59bd176eded74c8e941c9065850da4b1(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:42,822 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:42,822 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., storeName=d9a1bb9b05a472888bd945369e4af06a/A, priority=13, startTime=1732743342371; duration=0sec 2024-11-27T21:35:42,823 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:42,823 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a1bb9b05a472888bd945369e4af06a:A 2024-11-27T21:35:42,823 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:35:42,823 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): d9a1bb9b05a472888bd945369e4af06a/C is initiating minor compaction (all files) 2024-11-27T21:35:42,824 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a1bb9b05a472888bd945369e4af06a/C in TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:42,824 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/7a11115efbe24efe88f29e54eaaca27b, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/e427ef3b0ca54ce19369371ebd81a1b9, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/ef4aef0851454354a71f0b920672649f] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp, totalSize=36.7 K 2024-11-27T21:35:42,824 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a11115efbe24efe88f29e54eaaca27b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732743338579 2024-11-27T21:35:42,824 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting e427ef3b0ca54ce19369371ebd81a1b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732743339732 2024-11-27T21:35:42,825 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting ef4aef0851454354a71f0b920672649f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1732743341894 2024-11-27T21:35:42,832 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a1bb9b05a472888bd945369e4af06a#C#compaction#279 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:42,833 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/4b987fef02ff435d823f54df760cd1d2 is 50, key is test_row_0/C:col10/1732743341896/Put/seqid=0 2024-11-27T21:35:42,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742148_1324 (size=13051) 2024-11-27T21:35:42,849 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/4b987fef02ff435d823f54df760cd1d2 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/4b987fef02ff435d823f54df760cd1d2 2024-11-27T21:35:42,853 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9a1bb9b05a472888bd945369e4af06a/C of d9a1bb9b05a472888bd945369e4af06a into 4b987fef02ff435d823f54df760cd1d2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:42,853 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:42,853 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., storeName=d9a1bb9b05a472888bd945369e4af06a/C, priority=13, startTime=1732743342371; duration=0sec 2024-11-27T21:35:42,853 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:42,853 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a1bb9b05a472888bd945369e4af06a:C 2024-11-27T21:35:42,899 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/16a358771b7847eba3ff9b4dd5c413c1 2024-11-27T21:35:42,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/8020c759942a47e0a347f15ff5555431 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/8020c759942a47e0a347f15ff5555431 2024-11-27T21:35:42,908 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/8020c759942a47e0a347f15ff5555431, entries=150, sequenceid=328, filesize=12.0 K 2024-11-27T21:35:42,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/57e11db7a2eb49db98ff58abd49b3604 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/57e11db7a2eb49db98ff58abd49b3604 2024-11-27T21:35:42,912 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/57e11db7a2eb49db98ff58abd49b3604, entries=150, sequenceid=328, filesize=12.0 K 2024-11-27T21:35:42,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/16a358771b7847eba3ff9b4dd5c413c1 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/16a358771b7847eba3ff9b4dd5c413c1 2024-11-27T21:35:42,920 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/16a358771b7847eba3ff9b4dd5c413c1, entries=150, sequenceid=328, filesize=12.0 K 2024-11-27T21:35:42,921 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for d9a1bb9b05a472888bd945369e4af06a in 527ms, sequenceid=328, compaction requested=false 2024-11-27T21:35:42,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2603): Flush status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:42,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:42,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=87 2024-11-27T21:35:42,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=87 2024-11-27T21:35:42,924 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-27T21:35:42,924 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 985 msec 2024-11-27T21:35:42,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on d9a1bb9b05a472888bd945369e4af06a 2024-11-27T21:35:42,925 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:35:42,926 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d9a1bb9b05a472888bd945369e4af06a 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-27T21:35:42,927 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees in 990 msec 2024-11-27T21:35:42,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/303bd155d3c64e0ebd7d1f3b50903db5 is 50, key is test_row_0/A:col10/1732743342576/Put/seqid=0 2024-11-27T21:35:42,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742149_1325 (size=12301) 2024-11-27T21:35:42,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:42,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743402951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:42,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:42,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:42,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:42,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:42,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743402954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:42,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:42,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:42,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:42,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:42,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743402956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:42,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:42,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743402957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:42,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:42,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:42,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:42,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:42,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:42,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:43,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:43,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743403063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:43,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:43,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:43,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:43,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743403063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:43,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:43,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:43,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:43,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:43,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:43,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743403063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:43,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:43,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:43,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743403063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:43,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:43,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:43,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:43,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:43,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:43,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-27T21:35:43,074 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:35:43,074 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:35:43,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees 2024-11-27T21:35:43,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-27T21:35:43,076 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:35:43,077 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:35:43,077 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:35:43,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-27T21:35:43,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-27T21:35:43,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:43,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. as already flushing 2024-11-27T21:35:43,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:43,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:43,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:43,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:43,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:43,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743403273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:43,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:43,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:43,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:43,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:43,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743403273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:43,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:43,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:43,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:43,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:43,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743403274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:43,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:43,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:43,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743403274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:43,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:43,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:43,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:43,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:43,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:43,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/303bd155d3c64e0ebd7d1f3b50903db5 2024-11-27T21:35:43,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/862034abb2574aed9ed3721588d8d3ec is 50, key is test_row_0/B:col10/1732743342576/Put/seqid=0 2024-11-27T21:35:43,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742150_1326 (size=12301) 2024-11-27T21:35:43,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-27T21:35:43,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:43,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. as already flushing 2024-11-27T21:35:43,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:43,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:43,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:43,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:43,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-27T21:35:43,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-27T21:35:43,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:43,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. as already flushing 2024-11-27T21:35:43,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:43,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:43,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:43,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:43,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:43,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743403583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:43,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:43,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:43,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:43,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743403583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:43,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:43,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:43,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:43,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:43,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743403583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:43,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:43,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:43,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:43,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:43,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:43,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743403583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:43,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:43,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:43,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:43,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-27T21:35:43,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:43,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. as already flushing 2024-11-27T21:35:43,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:43,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:43,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:43,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:43,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-27T21:35:43,748 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/862034abb2574aed9ed3721588d8d3ec 2024-11-27T21:35:43,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/ad2ff2d43ed24a7a839f6cb794cde6f6 is 50, key is test_row_0/C:col10/1732743342576/Put/seqid=0 2024-11-27T21:35:43,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742151_1327 (size=12301) 2024-11-27T21:35:43,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-27T21:35:43,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:43,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. as already flushing 2024-11-27T21:35:43,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:43,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:43,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:43,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:43,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-27T21:35:43,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:43,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. as already flushing 2024-11-27T21:35:43,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:43,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:43,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:43,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:44,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:44,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:44,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743404093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:44,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743404093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:44,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:44,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743404093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:44,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:44,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743404093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:44,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-27T21:35:44,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:44,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. as already flushing 2024-11-27T21:35:44,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:44,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:44,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:44,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:44,159 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/ad2ff2d43ed24a7a839f6cb794cde6f6 2024-11-27T21:35:44,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/303bd155d3c64e0ebd7d1f3b50903db5 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/303bd155d3c64e0ebd7d1f3b50903db5 2024-11-27T21:35:44,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/303bd155d3c64e0ebd7d1f3b50903db5, entries=150, sequenceid=346, filesize=12.0 K 2024-11-27T21:35:44,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/862034abb2574aed9ed3721588d8d3ec as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/862034abb2574aed9ed3721588d8d3ec 2024-11-27T21:35:44,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/862034abb2574aed9ed3721588d8d3ec, entries=150, sequenceid=346, filesize=12.0 K 2024-11-27T21:35:44,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/ad2ff2d43ed24a7a839f6cb794cde6f6 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/ad2ff2d43ed24a7a839f6cb794cde6f6 2024-11-27T21:35:44,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/ad2ff2d43ed24a7a839f6cb794cde6f6, entries=150, sequenceid=346, filesize=12.0 K 2024-11-27T21:35:44,184 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for d9a1bb9b05a472888bd945369e4af06a in 1259ms, sequenceid=346, compaction requested=true 2024-11-27T21:35:44,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:44,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a1bb9b05a472888bd945369e4af06a:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:35:44,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:44,184 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:35:44,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a1bb9b05a472888bd945369e4af06a:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:35:44,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:44,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a1bb9b05a472888bd945369e4af06a:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:35:44,184 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:35:44,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:35:44,185 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:35:44,185 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): d9a1bb9b05a472888bd945369e4af06a/A is initiating minor compaction (all files) 2024-11-27T21:35:44,185 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a1bb9b05a472888bd945369e4af06a/A in TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:44,185 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/59bd176eded74c8e941c9065850da4b1, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/8020c759942a47e0a347f15ff5555431, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/303bd155d3c64e0ebd7d1f3b50903db5] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp, totalSize=36.8 K 2024-11-27T21:35:44,186 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:35:44,186 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): d9a1bb9b05a472888bd945369e4af06a/B is initiating minor compaction (all files) 2024-11-27T21:35:44,186 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a1bb9b05a472888bd945369e4af06a/B in TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:44,186 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59bd176eded74c8e941c9065850da4b1, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1732743341894 2024-11-27T21:35:44,186 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/563173fd116e474bb5a1e485f5248618, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/57e11db7a2eb49db98ff58abd49b3604, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/862034abb2574aed9ed3721588d8d3ec] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp, totalSize=36.8 K 2024-11-27T21:35:44,186 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 563173fd116e474bb5a1e485f5248618, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1732743341894 2024-11-27T21:35:44,186 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8020c759942a47e0a347f15ff5555431, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1732743341923 2024-11-27T21:35:44,186 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 57e11db7a2eb49db98ff58abd49b3604, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1732743341923 2024-11-27T21:35:44,187 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 303bd155d3c64e0ebd7d1f3b50903db5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732743342576 2024-11-27T21:35:44,187 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 862034abb2574aed9ed3721588d8d3ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732743342576 2024-11-27T21:35:44,194 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a1bb9b05a472888bd945369e4af06a#B#compaction#283 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:44,194 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a1bb9b05a472888bd945369e4af06a#A#compaction#284 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:44,194 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/c77e314f449a49c4b7d39b0022be99d2 is 50, key is test_row_0/B:col10/1732743342576/Put/seqid=0 2024-11-27T21:35:44,194 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/350045d446744083880dc715cc5ff3b6 is 50, key is test_row_0/A:col10/1732743342576/Put/seqid=0 2024-11-27T21:35:44,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742152_1328 (size=13153) 2024-11-27T21:35:44,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742153_1329 (size=13153) 2024-11-27T21:35:44,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-27T21:35:44,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-27T21:35:44,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:44,300 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2902): Flushing d9a1bb9b05a472888bd945369e4af06a 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-27T21:35:44,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/9ca1f44f461d4a9090e2c11c34e4c989 is 50, key is test_row_0/A:col10/1732743342952/Put/seqid=0 2024-11-27T21:35:44,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742154_1330 (size=12301) 2024-11-27T21:35:44,309 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/9ca1f44f461d4a9090e2c11c34e4c989 2024-11-27T21:35:44,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/f1d71d30d68742088a704312f06bad2b is 50, key is test_row_0/B:col10/1732743342952/Put/seqid=0 2024-11-27T21:35:44,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742155_1331 (size=12301) 2024-11-27T21:35:44,608 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/c77e314f449a49c4b7d39b0022be99d2 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/c77e314f449a49c4b7d39b0022be99d2 2024-11-27T21:35:44,610 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/350045d446744083880dc715cc5ff3b6 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/350045d446744083880dc715cc5ff3b6 2024-11-27T21:35:44,614 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9a1bb9b05a472888bd945369e4af06a/B of d9a1bb9b05a472888bd945369e4af06a into c77e314f449a49c4b7d39b0022be99d2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:44,614 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:44,614 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., storeName=d9a1bb9b05a472888bd945369e4af06a/B, priority=13, startTime=1732743344184; duration=0sec 2024-11-27T21:35:44,614 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:35:44,614 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a1bb9b05a472888bd945369e4af06a:B 2024-11-27T21:35:44,614 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:35:44,617 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:35:44,617 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): d9a1bb9b05a472888bd945369e4af06a/C is initiating minor compaction (all files) 2024-11-27T21:35:44,617 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a1bb9b05a472888bd945369e4af06a/C in TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:44,617 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/4b987fef02ff435d823f54df760cd1d2, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/16a358771b7847eba3ff9b4dd5c413c1, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/ad2ff2d43ed24a7a839f6cb794cde6f6] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp, totalSize=36.8 K 2024-11-27T21:35:44,617 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b987fef02ff435d823f54df760cd1d2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1732743341894 2024-11-27T21:35:44,617 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9a1bb9b05a472888bd945369e4af06a/A of d9a1bb9b05a472888bd945369e4af06a into 350045d446744083880dc715cc5ff3b6(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:44,617 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:44,617 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., storeName=d9a1bb9b05a472888bd945369e4af06a/A, priority=13, startTime=1732743344184; duration=0sec 2024-11-27T21:35:44,618 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:44,618 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a1bb9b05a472888bd945369e4af06a:A 2024-11-27T21:35:44,618 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 16a358771b7847eba3ff9b4dd5c413c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1732743341923 2024-11-27T21:35:44,618 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting ad2ff2d43ed24a7a839f6cb794cde6f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732743342576 2024-11-27T21:35:44,624 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a1bb9b05a472888bd945369e4af06a#C#compaction#287 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:44,625 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/f6b15b77c7eb4a488c99224a6c0e7d7f is 50, key is test_row_0/C:col10/1732743342576/Put/seqid=0 2024-11-27T21:35:44,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742156_1332 (size=13153) 2024-11-27T21:35:44,721 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/f1d71d30d68742088a704312f06bad2b 2024-11-27T21:35:44,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/10a5253be45c42a8abb34aef93ed5f79 is 50, key is test_row_0/C:col10/1732743342952/Put/seqid=0 2024-11-27T21:35:44,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742157_1333 (size=12301) 2024-11-27T21:35:45,044 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/f6b15b77c7eb4a488c99224a6c0e7d7f as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/f6b15b77c7eb4a488c99224a6c0e7d7f 2024-11-27T21:35:45,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on d9a1bb9b05a472888bd945369e4af06a 2024-11-27T21:35:45,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. as already flushing 2024-11-27T21:35:45,048 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9a1bb9b05a472888bd945369e4af06a/C of d9a1bb9b05a472888bd945369e4af06a into f6b15b77c7eb4a488c99224a6c0e7d7f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:45,048 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:45,048 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., storeName=d9a1bb9b05a472888bd945369e4af06a/C, priority=13, startTime=1732743344184; duration=0sec 2024-11-27T21:35:45,049 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:45,049 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a1bb9b05a472888bd945369e4af06a:C 2024-11-27T21:35:45,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:45,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60132 deadline: 1732743405090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:45,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:45,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:45,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:45,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:45,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732743405103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:45,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:45,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732743405103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:45,106 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,39097,1732743267048 for 'test_row_1' in d9a1bb9b05a472888bd945369e4af06a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:35:45,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:45,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:45,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:45,107 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,39097,1732743267048 for 'test_row_2' in d9a1bb9b05a472888bd945369e4af06a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:35:45,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:45,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:45,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:45,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:45,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60210 deadline: 1732743405113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:45,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:45,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60180 deadline: 1732743405113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:45,116 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,39097,1732743267048 for 'test_row_2' in d9a1bb9b05a472888bd945369e4af06a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:35:45,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:45,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:45,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:45,117 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,39097,1732743267048 for 'test_row_0' in d9a1bb9b05a472888bd945369e4af06a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:35:45,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:45,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:45,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:45,134 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/10a5253be45c42a8abb34aef93ed5f79 2024-11-27T21:35:45,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/9ca1f44f461d4a9090e2c11c34e4c989 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/9ca1f44f461d4a9090e2c11c34e4c989 2024-11-27T21:35:45,142 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/9ca1f44f461d4a9090e2c11c34e4c989, entries=150, sequenceid=367, filesize=12.0 K 2024-11-27T21:35:45,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/f1d71d30d68742088a704312f06bad2b as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/f1d71d30d68742088a704312f06bad2b 2024-11-27T21:35:45,147 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/f1d71d30d68742088a704312f06bad2b, entries=150, sequenceid=367, filesize=12.0 K 2024-11-27T21:35:45,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/10a5253be45c42a8abb34aef93ed5f79 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/10a5253be45c42a8abb34aef93ed5f79 2024-11-27T21:35:45,151 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/10a5253be45c42a8abb34aef93ed5f79, entries=150, sequenceid=367, filesize=12.0 K 2024-11-27T21:35:45,152 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for d9a1bb9b05a472888bd945369e4af06a in 852ms, sequenceid=367, compaction requested=false 2024-11-27T21:35:45,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2603): Flush status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:45,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:45,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=89 2024-11-27T21:35:45,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=89 2024-11-27T21:35:45,155 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-27T21:35:45,156 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0770 sec 2024-11-27T21:35:45,157 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees in 2.0820 sec 2024-11-27T21:35:45,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on d9a1bb9b05a472888bd945369e4af06a 2024-11-27T21:35:45,205 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:35:45,205 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d9a1bb9b05a472888bd945369e4af06a 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-27T21:35:45,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/70df89887e944f0984a7bdee01945eb7 is 50, key is test_row_0/A:col10/1732743345204/Put/seqid=0 2024-11-27T21:35:45,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742158_1334 (size=14741) 2024-11-27T21:35:45,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/70df89887e944f0984a7bdee01945eb7 2024-11-27T21:35:45,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/1eee536fe2d046599504018fc55a107b is 50, key is test_row_0/B:col10/1732743345204/Put/seqid=0 2024-11-27T21:35:45,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-27T21:35:45,224 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:35:45,224 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:35:45,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees 2024-11-27T21:35:45,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-27T21:35:45,227 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=90, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:35:45,228 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=90, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:35:45,228 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:35:45,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742159_1335 (size=12301) 2024-11-27T21:35:45,238 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/1eee536fe2d046599504018fc55a107b 2024-11-27T21:35:45,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/21c6f47a4c6144ac9ec9be79b861e00d is 50, key is test_row_0/C:col10/1732743345204/Put/seqid=0 2024-11-27T21:35:45,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742160_1336 (size=12301) 2024-11-27T21:35:45,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:45,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60132 deadline: 1732743405284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:45,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:45,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:45,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:45,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-27T21:35:45,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-27T21:35:45,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:45,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. as already flushing 2024-11-27T21:35:45,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:45,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:45,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:45,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:45,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:45,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60132 deadline: 1732743405393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:45,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:45,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:45,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:45,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-27T21:35:45,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:45,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. as already flushing 2024-11-27T21:35:45,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:45,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:45,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:45,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:45,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-27T21:35:45,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:45,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60132 deadline: 1732743405604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:45,607 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:45,607 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:45,607 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:45,649 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/21c6f47a4c6144ac9ec9be79b861e00d 2024-11-27T21:35:45,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/70df89887e944f0984a7bdee01945eb7 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/70df89887e944f0984a7bdee01945eb7 2024-11-27T21:35:45,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/70df89887e944f0984a7bdee01945eb7, entries=200, sequenceid=386, filesize=14.4 K 2024-11-27T21:35:45,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/1eee536fe2d046599504018fc55a107b as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/1eee536fe2d046599504018fc55a107b 2024-11-27T21:35:45,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/1eee536fe2d046599504018fc55a107b, entries=150, sequenceid=386, filesize=12.0 K 2024-11-27T21:35:45,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/21c6f47a4c6144ac9ec9be79b861e00d as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/21c6f47a4c6144ac9ec9be79b861e00d 2024-11-27T21:35:45,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/21c6f47a4c6144ac9ec9be79b861e00d, entries=150, sequenceid=386, filesize=12.0 K 2024-11-27T21:35:45,667 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for d9a1bb9b05a472888bd945369e4af06a in 462ms, sequenceid=386, compaction requested=true 2024-11-27T21:35:45,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:45,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a1bb9b05a472888bd945369e4af06a:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:35:45,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:45,667 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:35:45,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a1bb9b05a472888bd945369e4af06a:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:35:45,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:45,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a1bb9b05a472888bd945369e4af06a:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:35:45,667 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:35:45,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:35:45,668 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:35:45,668 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:35:45,668 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): d9a1bb9b05a472888bd945369e4af06a/A is initiating minor compaction (all files) 2024-11-27T21:35:45,668 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): d9a1bb9b05a472888bd945369e4af06a/B is initiating minor compaction (all files) 2024-11-27T21:35:45,668 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a1bb9b05a472888bd945369e4af06a/A in TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:45,668 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a1bb9b05a472888bd945369e4af06a/B in TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:45,668 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/350045d446744083880dc715cc5ff3b6, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/9ca1f44f461d4a9090e2c11c34e4c989, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/70df89887e944f0984a7bdee01945eb7] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp, totalSize=39.3 K 2024-11-27T21:35:45,668 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/c77e314f449a49c4b7d39b0022be99d2, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/f1d71d30d68742088a704312f06bad2b, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/1eee536fe2d046599504018fc55a107b] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp, totalSize=36.9 K 2024-11-27T21:35:45,669 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting c77e314f449a49c4b7d39b0022be99d2, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732743342576 2024-11-27T21:35:45,669 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 350045d446744083880dc715cc5ff3b6, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732743342576 2024-11-27T21:35:45,669 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting f1d71d30d68742088a704312f06bad2b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732743342952 2024-11-27T21:35:45,669 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ca1f44f461d4a9090e2c11c34e4c989, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732743342952 2024-11-27T21:35:45,669 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 1eee536fe2d046599504018fc55a107b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1732743345076 2024-11-27T21:35:45,669 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70df89887e944f0984a7bdee01945eb7, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1732743345076 2024-11-27T21:35:45,676 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a1bb9b05a472888bd945369e4af06a#B#compaction#292 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:45,677 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/9411ee60557544c5b994078031c2b4ce is 50, key is test_row_0/B:col10/1732743345204/Put/seqid=0 2024-11-27T21:35:45,677 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a1bb9b05a472888bd945369e4af06a#A#compaction#293 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:45,678 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/426f430440764669905fe473b297e502 is 50, key is test_row_0/A:col10/1732743345204/Put/seqid=0 2024-11-27T21:35:45,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742162_1338 (size=13255) 2024-11-27T21:35:45,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742161_1337 (size=13255) 2024-11-27T21:35:45,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-27T21:35:45,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:45,686 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2902): Flushing d9a1bb9b05a472888bd945369e4af06a 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-27T21:35:45,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/f6fe43364f0349ada2991e67eeb1669b is 50, key is test_row_0/A:col10/1732743345272/Put/seqid=0 2024-11-27T21:35:45,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742163_1339 (size=12301) 2024-11-27T21:35:45,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-27T21:35:45,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on d9a1bb9b05a472888bd945369e4af06a 2024-11-27T21:35:45,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. as already flushing 2024-11-27T21:35:45,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:45,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60132 deadline: 1732743405968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:45,972 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:45,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:45,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:46,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:46,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60132 deadline: 1732743406083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:46,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:46,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:46,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:46,091 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/9411ee60557544c5b994078031c2b4ce as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/9411ee60557544c5b994078031c2b4ce 2024-11-27T21:35:46,091 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/426f430440764669905fe473b297e502 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/426f430440764669905fe473b297e502 2024-11-27T21:35:46,095 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/f6fe43364f0349ada2991e67eeb1669b 2024-11-27T21:35:46,099 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9a1bb9b05a472888bd945369e4af06a/B of d9a1bb9b05a472888bd945369e4af06a into 9411ee60557544c5b994078031c2b4ce(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:46,099 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:46,099 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., storeName=d9a1bb9b05a472888bd945369e4af06a/B, priority=13, startTime=1732743345667; duration=0sec 2024-11-27T21:35:46,099 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:35:46,099 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a1bb9b05a472888bd945369e4af06a:B 2024-11-27T21:35:46,099 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:35:46,099 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9a1bb9b05a472888bd945369e4af06a/A of d9a1bb9b05a472888bd945369e4af06a into 426f430440764669905fe473b297e502(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:46,099 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:46,099 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., storeName=d9a1bb9b05a472888bd945369e4af06a/A, priority=13, startTime=1732743345667; duration=0sec 2024-11-27T21:35:46,099 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:46,099 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a1bb9b05a472888bd945369e4af06a:A 2024-11-27T21:35:46,100 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:35:46,100 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): d9a1bb9b05a472888bd945369e4af06a/C is initiating minor compaction (all files) 2024-11-27T21:35:46,100 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a1bb9b05a472888bd945369e4af06a/C in TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:46,100 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/f6b15b77c7eb4a488c99224a6c0e7d7f, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/10a5253be45c42a8abb34aef93ed5f79, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/21c6f47a4c6144ac9ec9be79b861e00d] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp, totalSize=36.9 K 2024-11-27T21:35:46,101 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting f6b15b77c7eb4a488c99224a6c0e7d7f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732743342576 2024-11-27T21:35:46,101 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 10a5253be45c42a8abb34aef93ed5f79, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732743342952 2024-11-27T21:35:46,101 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 21c6f47a4c6144ac9ec9be79b861e00d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1732743345076 2024-11-27T21:35:46,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/865b2263638546a99da137cf9dc20c1d is 50, key is test_row_0/B:col10/1732743345272/Put/seqid=0 2024-11-27T21:35:46,113 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a1bb9b05a472888bd945369e4af06a#C#compaction#296 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:46,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742164_1340 (size=12301) 2024-11-27T21:35:46,114 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/7036cc8c1caa4629b4097a32c1445360 is 50, key is test_row_0/C:col10/1732743345204/Put/seqid=0 2024-11-27T21:35:46,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742165_1341 (size=13255) 2024-11-27T21:35:46,126 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/7036cc8c1caa4629b4097a32c1445360 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/7036cc8c1caa4629b4097a32c1445360 2024-11-27T21:35:46,136 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9a1bb9b05a472888bd945369e4af06a/C of d9a1bb9b05a472888bd945369e4af06a into 7036cc8c1caa4629b4097a32c1445360(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:46,136 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:46,136 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., storeName=d9a1bb9b05a472888bd945369e4af06a/C, priority=13, startTime=1732743345667; duration=0sec 2024-11-27T21:35:46,136 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:46,136 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a1bb9b05a472888bd945369e4af06a:C 2024-11-27T21:35:46,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:46,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60132 deadline: 1732743406293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:46,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:46,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:46,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:46,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-27T21:35:46,514 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/865b2263638546a99da137cf9dc20c1d 2024-11-27T21:35:46,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/9df5f1a2201f4e7f86f73aeac47d2a42 is 50, key is test_row_0/C:col10/1732743345272/Put/seqid=0 2024-11-27T21:35:46,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742166_1342 (size=12301) 2024-11-27T21:35:46,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:46,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60132 deadline: 1732743406603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:46,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:46,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:46,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:35:46,749 INFO [Thread-1183 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1183. 2024-11-27T21:35:46,749 DEBUG [Thread-1183 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:35:46,749 DEBUG [Thread-1183 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:46,749 DEBUG [Thread-1183 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:46,749 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:35:46,750 INFO [Thread-1181 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1181. 2024-11-27T21:35:46,750 DEBUG [Thread-1181 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:35:46,750 DEBUG [Thread-1181 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:46,750 INFO [Thread-1177 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1177. 2024-11-27T21:35:46,750 DEBUG [Thread-1177 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:35:46,750 DEBUG [Thread-1181 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:46,750 DEBUG [Thread-1177 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:46,750 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:35:46,750 DEBUG [Thread-1177 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:46,750 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:35:46,753 INFO [Thread-1179 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1179. 2024-11-27T21:35:46,753 DEBUG [Thread-1179 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:35:46,753 DEBUG [Thread-1179 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:46,753 INFO [Thread-1175 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1175. 2024-11-27T21:35:46,753 DEBUG [Thread-1175 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:35:46,753 DEBUG [Thread-1175 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:46,753 DEBUG [Thread-1179 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:46,753 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:35:46,753 DEBUG [Thread-1175 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:46,753 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:35:46,926 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/9df5f1a2201f4e7f86f73aeac47d2a42 2024-11-27T21:35:46,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/f6fe43364f0349ada2991e67eeb1669b as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/f6fe43364f0349ada2991e67eeb1669b 2024-11-27T21:35:46,933 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/f6fe43364f0349ada2991e67eeb1669b, entries=150, sequenceid=406, filesize=12.0 K 2024-11-27T21:35:46,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/865b2263638546a99da137cf9dc20c1d as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/865b2263638546a99da137cf9dc20c1d 2024-11-27T21:35:46,936 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/865b2263638546a99da137cf9dc20c1d, entries=150, sequenceid=406, filesize=12.0 K 2024-11-27T21:35:46,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/9df5f1a2201f4e7f86f73aeac47d2a42 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/9df5f1a2201f4e7f86f73aeac47d2a42 2024-11-27T21:35:46,939 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/9df5f1a2201f4e7f86f73aeac47d2a42, entries=150, sequenceid=406, filesize=12.0 K 2024-11-27T21:35:46,940 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for d9a1bb9b05a472888bd945369e4af06a in 1254ms, sequenceid=406, compaction requested=false 2024-11-27T21:35:46,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2603): Flush status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:46,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:46,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=91 2024-11-27T21:35:46,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=91 2024-11-27T21:35:46,943 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-27T21:35:46,943 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7140 sec 2024-11-27T21:35:46,944 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees in 1.7190 sec 2024-11-27T21:35:47,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on d9a1bb9b05a472888bd945369e4af06a 2024-11-27T21:35:47,114 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:35:47,115 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d9a1bb9b05a472888bd945369e4af06a 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-27T21:35:47,115 INFO [Thread-1166 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1166. 2024-11-27T21:35:47,115 INFO [Thread-1170 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1170. 2024-11-27T21:35:47,115 INFO [Thread-1168 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1168. 2024-11-27T21:35:47,115 DEBUG [Thread-1166 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:35:47,115 DEBUG [Thread-1170 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:35:47,115 DEBUG [Thread-1166 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:47,115 DEBUG [Thread-1168 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:35:47,115 DEBUG [Thread-1168 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:47,115 DEBUG [Thread-1170 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:47,115 DEBUG [Thread-1166 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:47,115 DEBUG [Thread-1168 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:47,115 DEBUG [Thread-1170 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:47,115 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:35:47,115 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:35:47,115 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:35:47,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/906372d93f144227ab434a8ff5e64a06 is 50, key is test_row_0/A:col10/1732743345967/Put/seqid=0 2024-11-27T21:35:47,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742167_1343 (size=12301) 2024-11-27T21:35:47,135 INFO [Thread-1164 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1164. 2024-11-27T21:35:47,135 DEBUG [Thread-1164 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:35:47,135 DEBUG [Thread-1164 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:47,135 DEBUG [Thread-1164 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:47,135 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:35:47,145 INFO [Thread-1172 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1172. 2024-11-27T21:35:47,145 DEBUG [Thread-1172 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:35:47,145 DEBUG [Thread-1172 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:47,145 DEBUG [Thread-1172 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:47,145 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:35:47,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-27T21:35:47,374 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:35:47,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-27T21:35:47,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-27T21:35:47,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-27T21:35:47,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-27T21:35:47,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-27T21:35:47,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-11-27T21:35:47,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-27T21:35:47,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3774 2024-11-27T21:35:47,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3743 2024-11-27T21:35:47,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3722 2024-11-27T21:35:47,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3773 2024-11-27T21:35:47,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3733 2024-11-27T21:35:47,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-27T21:35:47,374 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T21:35:47,374 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T21:35:47,374 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:47,374 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:47,374 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:35:47,375 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-27T21:35:47,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-27T21:35:47,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-27T21:35:47,378 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732743347377"}]},"ts":"1732743347377"} 2024-11-27T21:35:47,379 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-27T21:35:47,379 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-27T21:35:47,379 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T21:35:47,381 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d9a1bb9b05a472888bd945369e4af06a, UNASSIGN}] 2024-11-27T21:35:47,381 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d9a1bb9b05a472888bd945369e4af06a, UNASSIGN 2024-11-27T21:35:47,382 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=d9a1bb9b05a472888bd945369e4af06a, regionState=CLOSING, regionLocation=4585c39bed03,39097,1732743267048 2024-11-27T21:35:47,383 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d9a1bb9b05a472888bd945369e4af06a, UNASSIGN because future has completed 2024-11-27T21:35:47,383 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T21:35:47,383 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048}] 2024-11-27T21:35:47,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-27T21:35:47,523 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/906372d93f144227ab434a8ff5e64a06 2024-11-27T21:35:47,529 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/146c698b22054c5abb0627e5d59350ce is 50, key is test_row_0/B:col10/1732743345967/Put/seqid=0 2024-11-27T21:35:47,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742168_1344 (size=12301) 2024-11-27T21:35:47,535 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(122): Close d9a1bb9b05a472888bd945369e4af06a 2024-11-27T21:35:47,535 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T21:35:47,535 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1722): Closing d9a1bb9b05a472888bd945369e4af06a, disabling compactions & flushes 2024-11-27T21:35:47,535 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:47,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-27T21:35:47,932 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/146c698b22054c5abb0627e5d59350ce 2024-11-27T21:35:47,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/7c197dfa054841048cbbc292c592dcf3 is 50, key is test_row_0/C:col10/1732743345967/Put/seqid=0 2024-11-27T21:35:47,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742169_1345 (size=12301) 2024-11-27T21:35:48,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-27T21:35:48,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/7c197dfa054841048cbbc292c592dcf3 2024-11-27T21:35:48,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/906372d93f144227ab434a8ff5e64a06 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/906372d93f144227ab434a8ff5e64a06 2024-11-27T21:35:48,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/906372d93f144227ab434a8ff5e64a06, entries=150, sequenceid=428, filesize=12.0 K 2024-11-27T21:35:48,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/146c698b22054c5abb0627e5d59350ce as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/146c698b22054c5abb0627e5d59350ce 2024-11-27T21:35:48,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/146c698b22054c5abb0627e5d59350ce, entries=150, sequenceid=428, filesize=12.0 K 2024-11-27T21:35:48,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/7c197dfa054841048cbbc292c592dcf3 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/7c197dfa054841048cbbc292c592dcf3 2024-11-27T21:35:48,355 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/7c197dfa054841048cbbc292c592dcf3, entries=150, sequenceid=428, filesize=12.0 K 2024-11-27T21:35:48,356 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=13.42 KB/13740 for d9a1bb9b05a472888bd945369e4af06a in 1242ms, sequenceid=428, compaction requested=true 2024-11-27T21:35:48,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d9a1bb9b05a472888bd945369e4af06a: 2024-11-27T21:35:48,356 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:48,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a1bb9b05a472888bd945369e4af06a:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:35:48,356 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:48,356 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. because compaction request was cancelled 2024-11-27T21:35:48,356 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. after waiting 0 ms 2024-11-27T21:35:48,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:48,356 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a1bb9b05a472888bd945369e4af06a:A 2024-11-27T21:35:48,356 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:48,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a1bb9b05a472888bd945369e4af06a:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:35:48,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:48,356 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. because compaction request was cancelled 2024-11-27T21:35:48,356 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a1bb9b05a472888bd945369e4af06a:B 2024-11-27T21:35:48,356 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(2902): Flushing d9a1bb9b05a472888bd945369e4af06a 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-27T21:35:48,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a1bb9b05a472888bd945369e4af06a:C, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:35:48,356 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. because compaction request was cancelled 2024-11-27T21:35:48,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:48,356 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a1bb9b05a472888bd945369e4af06a:C 2024-11-27T21:35:48,359 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/bc0510aebd624524b47d54f3a886efb7 is 50, key is test_row_0/A:col10/1732743347144/Put/seqid=0 2024-11-27T21:35:48,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742170_1346 (size=9857) 2024-11-27T21:35:48,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-27T21:35:48,763 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/bc0510aebd624524b47d54f3a886efb7 2024-11-27T21:35:48,769 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/b0f7de06b5f143d8839ad6cdd27163ef is 50, key is test_row_0/B:col10/1732743347144/Put/seqid=0 2024-11-27T21:35:48,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742171_1347 (size=9857) 2024-11-27T21:35:49,173 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/b0f7de06b5f143d8839ad6cdd27163ef 2024-11-27T21:35:49,179 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/c44dbf6811cd474987c7637b23fb4de0 is 50, key is test_row_0/C:col10/1732743347144/Put/seqid=0 2024-11-27T21:35:49,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742172_1348 (size=9857) 2024-11-27T21:35:49,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-27T21:35:49,582 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/c44dbf6811cd474987c7637b23fb4de0 2024-11-27T21:35:49,586 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/A/bc0510aebd624524b47d54f3a886efb7 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/bc0510aebd624524b47d54f3a886efb7 2024-11-27T21:35:49,589 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/bc0510aebd624524b47d54f3a886efb7, entries=100, sequenceid=433, filesize=9.6 K 2024-11-27T21:35:49,590 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/B/b0f7de06b5f143d8839ad6cdd27163ef as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/b0f7de06b5f143d8839ad6cdd27163ef 2024-11-27T21:35:49,593 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/b0f7de06b5f143d8839ad6cdd27163ef, entries=100, sequenceid=433, filesize=9.6 K 2024-11-27T21:35:49,593 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/.tmp/C/c44dbf6811cd474987c7637b23fb4de0 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/c44dbf6811cd474987c7637b23fb4de0 2024-11-27T21:35:49,596 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/c44dbf6811cd474987c7637b23fb4de0, entries=100, sequenceid=433, filesize=9.6 K 2024-11-27T21:35:49,597 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for d9a1bb9b05a472888bd945369e4af06a in 1241ms, sequenceid=433, compaction requested=true 2024-11-27T21:35:49,598 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/d7765506bb4148a38eff6b4b1260ad90, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/38bcab1c2a044387962cba88c8b9a3bc, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/27b0a04af5794db48bf808090b614ab7, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/4df7da5891e34aa6a9354358985286d3, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/a4161037ddbf40ff84096e4b6562700b, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/8145be8efda04281a8a615d4b194e1ed, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/0726c1d6bafe4772a99f78a875161ea5, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/ad21ab174ef64322b62aaeebfc7bc0da, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/722d09fb152145449760b3ed1a399a66, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/50e03acc026743a092d7d87aacfe80e6, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/b7b34394407a4fcb92018f884903559a, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/1c31b61189994ad78e8b0f7ccdbd0ebc, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/a263947548b54002acecd65192ff7652, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/dc99716cd61e4c5b89d65d43f7a4dd53, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/f97ca61bb8794672a65b4ef7e8a2f002, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/71022d59ede146098e5799bd0d5ad945, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/5d3eeb9f8aea4724a7386a4f7d75964d, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/454cc23617e0415a99d8127435b6fde9, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/d9d0dde632974ee89ef38729c9a9aae1, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/2de6feb3121f48578aa79c84db65e2e6, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/171b6ffda9fd4634a8b7c2c6bd7a9508, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/445828923fe94b69a5958ed88e894924, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/59bd176eded74c8e941c9065850da4b1, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/8020c759942a47e0a347f15ff5555431, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/350045d446744083880dc715cc5ff3b6, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/303bd155d3c64e0ebd7d1f3b50903db5, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/9ca1f44f461d4a9090e2c11c34e4c989, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/70df89887e944f0984a7bdee01945eb7] to archive 2024-11-27T21:35:49,598 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T21:35:49,600 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/d7765506bb4148a38eff6b4b1260ad90 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/d7765506bb4148a38eff6b4b1260ad90 2024-11-27T21:35:49,601 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/38bcab1c2a044387962cba88c8b9a3bc to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/38bcab1c2a044387962cba88c8b9a3bc 2024-11-27T21:35:49,602 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/27b0a04af5794db48bf808090b614ab7 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/27b0a04af5794db48bf808090b614ab7 2024-11-27T21:35:49,603 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/4df7da5891e34aa6a9354358985286d3 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/4df7da5891e34aa6a9354358985286d3 2024-11-27T21:35:49,603 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/a4161037ddbf40ff84096e4b6562700b to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/a4161037ddbf40ff84096e4b6562700b 2024-11-27T21:35:49,604 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/8145be8efda04281a8a615d4b194e1ed to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/8145be8efda04281a8a615d4b194e1ed 2024-11-27T21:35:49,605 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/0726c1d6bafe4772a99f78a875161ea5 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/0726c1d6bafe4772a99f78a875161ea5 2024-11-27T21:35:49,606 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/ad21ab174ef64322b62aaeebfc7bc0da to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/ad21ab174ef64322b62aaeebfc7bc0da 2024-11-27T21:35:49,607 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/722d09fb152145449760b3ed1a399a66 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/722d09fb152145449760b3ed1a399a66 2024-11-27T21:35:49,608 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/50e03acc026743a092d7d87aacfe80e6 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/50e03acc026743a092d7d87aacfe80e6 2024-11-27T21:35:49,609 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/b7b34394407a4fcb92018f884903559a to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/b7b34394407a4fcb92018f884903559a 2024-11-27T21:35:49,609 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/1c31b61189994ad78e8b0f7ccdbd0ebc to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/1c31b61189994ad78e8b0f7ccdbd0ebc 2024-11-27T21:35:49,610 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/a263947548b54002acecd65192ff7652 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/a263947548b54002acecd65192ff7652 2024-11-27T21:35:49,611 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/dc99716cd61e4c5b89d65d43f7a4dd53 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/dc99716cd61e4c5b89d65d43f7a4dd53 2024-11-27T21:35:49,612 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/f97ca61bb8794672a65b4ef7e8a2f002 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/f97ca61bb8794672a65b4ef7e8a2f002 2024-11-27T21:35:49,613 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/71022d59ede146098e5799bd0d5ad945 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/71022d59ede146098e5799bd0d5ad945 2024-11-27T21:35:49,614 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/5d3eeb9f8aea4724a7386a4f7d75964d to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/5d3eeb9f8aea4724a7386a4f7d75964d 2024-11-27T21:35:49,615 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/454cc23617e0415a99d8127435b6fde9 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/454cc23617e0415a99d8127435b6fde9 2024-11-27T21:35:49,615 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/d9d0dde632974ee89ef38729c9a9aae1 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/d9d0dde632974ee89ef38729c9a9aae1 2024-11-27T21:35:49,616 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/2de6feb3121f48578aa79c84db65e2e6 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/2de6feb3121f48578aa79c84db65e2e6 2024-11-27T21:35:49,617 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/171b6ffda9fd4634a8b7c2c6bd7a9508 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/171b6ffda9fd4634a8b7c2c6bd7a9508 2024-11-27T21:35:49,618 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/445828923fe94b69a5958ed88e894924 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/445828923fe94b69a5958ed88e894924 2024-11-27T21:35:49,619 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/59bd176eded74c8e941c9065850da4b1 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/59bd176eded74c8e941c9065850da4b1 2024-11-27T21:35:49,620 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/8020c759942a47e0a347f15ff5555431 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/8020c759942a47e0a347f15ff5555431 2024-11-27T21:35:49,621 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/350045d446744083880dc715cc5ff3b6 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/350045d446744083880dc715cc5ff3b6 2024-11-27T21:35:49,622 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/303bd155d3c64e0ebd7d1f3b50903db5 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/303bd155d3c64e0ebd7d1f3b50903db5 2024-11-27T21:35:49,622 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/9ca1f44f461d4a9090e2c11c34e4c989 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/9ca1f44f461d4a9090e2c11c34e4c989 2024-11-27T21:35:49,624 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/70df89887e944f0984a7bdee01945eb7 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/70df89887e944f0984a7bdee01945eb7 2024-11-27T21:35:49,625 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/1d750cadfef7471bb1cb55813e54a68e, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/dc681db814a442f594705b40596c2a74, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/9b0dfa0f5065461eb3524c922bf6db3d, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/ea0fe727a732402fa92a3c1cebf15ea5, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/f793410708a6497faec41231722caa11, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/9fca3a3c2c464cfb9693d4edae0f2e32, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/d77aaa06fb2449a18752ad280db78f95, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/ee6563bd37784e308cecf445842df054, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/2aa5d24831ab421397dbcbe8cfa4a350, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/6e27d578c74f4189af1ac5188a42df63, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/7482c3589fb54e5ea61626332319485a, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/fc15cb4cc6d648dcaf86e2b390796b4b, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/c576f6e080094a80ba1a5384c4948823, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/2708fd3769a24fbf84705fa79106dfaa, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/0a6d639af7da4edf836fb66a9e0bbf82, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/a1655af8e8694749be447ade6df04777, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/b435466a7b94451686b6cb07af2f223a, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/0b164efbef54486e892435e497a13a79, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/b8a5624851ca4e9081379ba2c462e592, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/f43dc7d8eaca45ab8282b536f478ea55, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/2c7a0b13b0fb453b829101525a7155c8, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/563173fd116e474bb5a1e485f5248618, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/557bbc93ca904e8c8794533ba171392d, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/57e11db7a2eb49db98ff58abd49b3604, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/c77e314f449a49c4b7d39b0022be99d2, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/862034abb2574aed9ed3721588d8d3ec, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/f1d71d30d68742088a704312f06bad2b, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/1eee536fe2d046599504018fc55a107b] to archive 2024-11-27T21:35:49,626 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T21:35:49,627 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/1d750cadfef7471bb1cb55813e54a68e to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/1d750cadfef7471bb1cb55813e54a68e 2024-11-27T21:35:49,628 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/dc681db814a442f594705b40596c2a74 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/dc681db814a442f594705b40596c2a74 2024-11-27T21:35:49,628 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/9b0dfa0f5065461eb3524c922bf6db3d to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/9b0dfa0f5065461eb3524c922bf6db3d 2024-11-27T21:35:49,629 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/ea0fe727a732402fa92a3c1cebf15ea5 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/ea0fe727a732402fa92a3c1cebf15ea5 2024-11-27T21:35:49,630 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/f793410708a6497faec41231722caa11 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/f793410708a6497faec41231722caa11 2024-11-27T21:35:49,631 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/9fca3a3c2c464cfb9693d4edae0f2e32 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/9fca3a3c2c464cfb9693d4edae0f2e32 2024-11-27T21:35:49,632 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/d77aaa06fb2449a18752ad280db78f95 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/d77aaa06fb2449a18752ad280db78f95 2024-11-27T21:35:49,633 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/ee6563bd37784e308cecf445842df054 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/ee6563bd37784e308cecf445842df054 2024-11-27T21:35:49,634 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/2aa5d24831ab421397dbcbe8cfa4a350 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/2aa5d24831ab421397dbcbe8cfa4a350 2024-11-27T21:35:49,635 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/6e27d578c74f4189af1ac5188a42df63 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/6e27d578c74f4189af1ac5188a42df63 2024-11-27T21:35:49,636 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/7482c3589fb54e5ea61626332319485a to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/7482c3589fb54e5ea61626332319485a 2024-11-27T21:35:49,637 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/fc15cb4cc6d648dcaf86e2b390796b4b to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/fc15cb4cc6d648dcaf86e2b390796b4b 2024-11-27T21:35:49,637 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/c576f6e080094a80ba1a5384c4948823 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/c576f6e080094a80ba1a5384c4948823 2024-11-27T21:35:49,638 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/2708fd3769a24fbf84705fa79106dfaa to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/2708fd3769a24fbf84705fa79106dfaa 2024-11-27T21:35:49,639 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/0a6d639af7da4edf836fb66a9e0bbf82 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/0a6d639af7da4edf836fb66a9e0bbf82 2024-11-27T21:35:49,640 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/a1655af8e8694749be447ade6df04777 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/a1655af8e8694749be447ade6df04777 2024-11-27T21:35:49,641 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/b435466a7b94451686b6cb07af2f223a to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/b435466a7b94451686b6cb07af2f223a 2024-11-27T21:35:49,641 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/0b164efbef54486e892435e497a13a79 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/0b164efbef54486e892435e497a13a79 2024-11-27T21:35:49,642 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/b8a5624851ca4e9081379ba2c462e592 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/b8a5624851ca4e9081379ba2c462e592 2024-11-27T21:35:49,643 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/f43dc7d8eaca45ab8282b536f478ea55 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/f43dc7d8eaca45ab8282b536f478ea55 2024-11-27T21:35:49,644 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/2c7a0b13b0fb453b829101525a7155c8 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/2c7a0b13b0fb453b829101525a7155c8 2024-11-27T21:35:49,645 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/563173fd116e474bb5a1e485f5248618 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/563173fd116e474bb5a1e485f5248618 2024-11-27T21:35:49,646 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/557bbc93ca904e8c8794533ba171392d to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/557bbc93ca904e8c8794533ba171392d 2024-11-27T21:35:49,647 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/57e11db7a2eb49db98ff58abd49b3604 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/57e11db7a2eb49db98ff58abd49b3604 2024-11-27T21:35:49,647 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/c77e314f449a49c4b7d39b0022be99d2 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/c77e314f449a49c4b7d39b0022be99d2 2024-11-27T21:35:49,648 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/862034abb2574aed9ed3721588d8d3ec to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/862034abb2574aed9ed3721588d8d3ec 2024-11-27T21:35:49,649 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/f1d71d30d68742088a704312f06bad2b to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/f1d71d30d68742088a704312f06bad2b 2024-11-27T21:35:49,650 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/1eee536fe2d046599504018fc55a107b to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/1eee536fe2d046599504018fc55a107b 2024-11-27T21:35:49,651 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/82e1771ea7764d6b85e9438f63ed42bd, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/7f005a32ef4240ceb1d3d43eeb97476a, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/3c8093c213324d66ae9ce1f8a8b069a6, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/7b61fd7b494a48f6ac179a1f063419a0, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/a7eb42d75bb2430c811170f31e6e5e21, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/eb8f04fca954401a8eb1f9c0b4099853, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/f1be662485e14b04ae62a375f64ddeff, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/7667452d652b48688148a1a858751c29, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/ad8d4284f72544eea73700368529be18, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/c2c5dfef8e9643dd91337635c730735a, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/26ccb361a1d24b7ab82d5ddf52fb2e68, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/8f0dc41f8d934f4da7213a51afe29f11, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/0513dd2a4cd140dc9a4b42e3472bf730, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/d5b4c85fdcca40f896c44891cb999a40, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/1289e38d20d64b6fb4c7085a98f0397f, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/0cdd4cd911504da7a88dc99c44e2194a, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/74ef579ef94147b484428e69f3733619, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/dad9770f1db04e6a8ff54634185244de, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/7a11115efbe24efe88f29e54eaaca27b, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/990241e5cd1d4507b13ed61cb9e01823, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/e427ef3b0ca54ce19369371ebd81a1b9, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/4b987fef02ff435d823f54df760cd1d2, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/ef4aef0851454354a71f0b920672649f, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/16a358771b7847eba3ff9b4dd5c413c1, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/f6b15b77c7eb4a488c99224a6c0e7d7f, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/ad2ff2d43ed24a7a839f6cb794cde6f6, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/10a5253be45c42a8abb34aef93ed5f79, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/21c6f47a4c6144ac9ec9be79b861e00d] to archive 2024-11-27T21:35:49,652 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T21:35:49,653 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/82e1771ea7764d6b85e9438f63ed42bd to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/82e1771ea7764d6b85e9438f63ed42bd 2024-11-27T21:35:49,654 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/7f005a32ef4240ceb1d3d43eeb97476a to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/7f005a32ef4240ceb1d3d43eeb97476a 2024-11-27T21:35:49,655 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/3c8093c213324d66ae9ce1f8a8b069a6 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/3c8093c213324d66ae9ce1f8a8b069a6 2024-11-27T21:35:49,656 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/7b61fd7b494a48f6ac179a1f063419a0 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/7b61fd7b494a48f6ac179a1f063419a0 2024-11-27T21:35:49,656 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/a7eb42d75bb2430c811170f31e6e5e21 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/a7eb42d75bb2430c811170f31e6e5e21 2024-11-27T21:35:49,657 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/eb8f04fca954401a8eb1f9c0b4099853 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/eb8f04fca954401a8eb1f9c0b4099853 2024-11-27T21:35:49,658 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/f1be662485e14b04ae62a375f64ddeff to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/f1be662485e14b04ae62a375f64ddeff 2024-11-27T21:35:49,659 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/7667452d652b48688148a1a858751c29 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/7667452d652b48688148a1a858751c29 2024-11-27T21:35:49,659 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/ad8d4284f72544eea73700368529be18 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/ad8d4284f72544eea73700368529be18 2024-11-27T21:35:49,660 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/c2c5dfef8e9643dd91337635c730735a to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/c2c5dfef8e9643dd91337635c730735a 2024-11-27T21:35:49,661 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/26ccb361a1d24b7ab82d5ddf52fb2e68 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/26ccb361a1d24b7ab82d5ddf52fb2e68 2024-11-27T21:35:49,662 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/8f0dc41f8d934f4da7213a51afe29f11 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/8f0dc41f8d934f4da7213a51afe29f11 2024-11-27T21:35:49,663 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/0513dd2a4cd140dc9a4b42e3472bf730 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/0513dd2a4cd140dc9a4b42e3472bf730 2024-11-27T21:35:49,663 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/d5b4c85fdcca40f896c44891cb999a40 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/d5b4c85fdcca40f896c44891cb999a40 2024-11-27T21:35:49,664 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/1289e38d20d64b6fb4c7085a98f0397f to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/1289e38d20d64b6fb4c7085a98f0397f 2024-11-27T21:35:49,665 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/0cdd4cd911504da7a88dc99c44e2194a to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/0cdd4cd911504da7a88dc99c44e2194a 2024-11-27T21:35:49,666 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/74ef579ef94147b484428e69f3733619 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/74ef579ef94147b484428e69f3733619 2024-11-27T21:35:49,667 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/dad9770f1db04e6a8ff54634185244de to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/dad9770f1db04e6a8ff54634185244de 2024-11-27T21:35:49,668 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/7a11115efbe24efe88f29e54eaaca27b to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/7a11115efbe24efe88f29e54eaaca27b 2024-11-27T21:35:49,669 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/990241e5cd1d4507b13ed61cb9e01823 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/990241e5cd1d4507b13ed61cb9e01823 2024-11-27T21:35:49,669 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/e427ef3b0ca54ce19369371ebd81a1b9 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/e427ef3b0ca54ce19369371ebd81a1b9 2024-11-27T21:35:49,670 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/4b987fef02ff435d823f54df760cd1d2 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/4b987fef02ff435d823f54df760cd1d2 2024-11-27T21:35:49,671 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/ef4aef0851454354a71f0b920672649f to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/ef4aef0851454354a71f0b920672649f 2024-11-27T21:35:49,672 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/16a358771b7847eba3ff9b4dd5c413c1 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/16a358771b7847eba3ff9b4dd5c413c1 2024-11-27T21:35:49,673 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/f6b15b77c7eb4a488c99224a6c0e7d7f to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/f6b15b77c7eb4a488c99224a6c0e7d7f 2024-11-27T21:35:49,673 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/ad2ff2d43ed24a7a839f6cb794cde6f6 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/ad2ff2d43ed24a7a839f6cb794cde6f6 2024-11-27T21:35:49,674 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/10a5253be45c42a8abb34aef93ed5f79 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/10a5253be45c42a8abb34aef93ed5f79 2024-11-27T21:35:49,675 DEBUG [StoreCloser-TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/21c6f47a4c6144ac9ec9be79b861e00d to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/21c6f47a4c6144ac9ec9be79b861e00d 2024-11-27T21:35:49,679 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/recovered.edits/436.seqid, newMaxSeqId=436, maxSeqId=1 2024-11-27T21:35:49,680 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. 2024-11-27T21:35:49,680 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1676): Region close journal for d9a1bb9b05a472888bd945369e4af06a: Waiting for close lock at 1732743347535Running coprocessor pre-close hooks at 1732743347535Disabling compacts and flushes for region at 1732743347535Disabling writes for close at 1732743348356 (+821 ms)Obtaining lock to block concurrent updates at 1732743348356Preparing flush snapshotting stores in d9a1bb9b05a472888bd945369e4af06a at 1732743348356Finished memstore snapshotting TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1732743348357 (+1 ms)Flushing stores of TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a. at 1732743348357Flushing d9a1bb9b05a472888bd945369e4af06a/A: creating writer at 1732743348357Flushing d9a1bb9b05a472888bd945369e4af06a/A: appending metadata at 1732743348359 (+2 ms)Flushing d9a1bb9b05a472888bd945369e4af06a/A: closing flushed file at 1732743348359Flushing d9a1bb9b05a472888bd945369e4af06a/B: creating writer at 1732743348767 (+408 ms)Flushing d9a1bb9b05a472888bd945369e4af06a/B: appending metadata at 1732743348769 (+2 ms)Flushing d9a1bb9b05a472888bd945369e4af06a/B: closing flushed file at 1732743348769Flushing d9a1bb9b05a472888bd945369e4af06a/C: creating writer at 1732743349176 (+407 ms)Flushing d9a1bb9b05a472888bd945369e4af06a/C: appending metadata at 1732743349178 (+2 ms)Flushing d9a1bb9b05a472888bd945369e4af06a/C: closing flushed file at 1732743349178Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13138972: reopening flushed file at 1732743349586 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34b30d3: reopening flushed file at 1732743349589 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@546b7545: reopening flushed file at 1732743349593 (+4 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for d9a1bb9b05a472888bd945369e4af06a in 1241ms, sequenceid=433, compaction requested=true at 1732743349597 (+4 ms)Writing region close event to WAL at 1732743349676 (+79 ms)Running coprocessor post-close hooks at 1732743349679 (+3 ms)Closed at 1732743349680 (+1 ms) 2024-11-27T21:35:49,681 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(157): Closed d9a1bb9b05a472888bd945369e4af06a 2024-11-27T21:35:49,682 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=d9a1bb9b05a472888bd945369e4af06a, regionState=CLOSED 2024-11-27T21:35:49,683 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 because future has completed 2024-11-27T21:35:49,685 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-27T21:35:49,685 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; CloseRegionProcedure d9a1bb9b05a472888bd945369e4af06a, server=4585c39bed03,39097,1732743267048 in 2.3010 sec 2024-11-27T21:35:49,686 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-27T21:35:49,687 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d9a1bb9b05a472888bd945369e4af06a, UNASSIGN in 2.3040 sec 2024-11-27T21:35:49,688 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-27T21:35:49,688 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.3080 sec 2024-11-27T21:35:49,690 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732743349689"}]},"ts":"1732743349689"} 2024-11-27T21:35:49,691 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-27T21:35:49,691 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-27T21:35:49,692 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.3160 sec 2024-11-27T21:35:51,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-27T21:35:51,544 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:35:51,544 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-27T21:35:51,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=96, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:35:51,546 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=96, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:35:51,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-27T21:35:51,546 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=96, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:35:51,549 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a 2024-11-27T21:35:51,551 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A, FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B, FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C, FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/recovered.edits] 2024-11-27T21:35:51,553 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/426f430440764669905fe473b297e502 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/426f430440764669905fe473b297e502 2024-11-27T21:35:51,554 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/906372d93f144227ab434a8ff5e64a06 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/906372d93f144227ab434a8ff5e64a06 2024-11-27T21:35:51,555 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/bc0510aebd624524b47d54f3a886efb7 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/bc0510aebd624524b47d54f3a886efb7 2024-11-27T21:35:51,555 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/f6fe43364f0349ada2991e67eeb1669b to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/A/f6fe43364f0349ada2991e67eeb1669b 2024-11-27T21:35:51,557 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/146c698b22054c5abb0627e5d59350ce to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/146c698b22054c5abb0627e5d59350ce 2024-11-27T21:35:51,558 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/865b2263638546a99da137cf9dc20c1d to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/865b2263638546a99da137cf9dc20c1d 2024-11-27T21:35:51,560 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/9411ee60557544c5b994078031c2b4ce to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/9411ee60557544c5b994078031c2b4ce 2024-11-27T21:35:51,561 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/b0f7de06b5f143d8839ad6cdd27163ef to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/B/b0f7de06b5f143d8839ad6cdd27163ef 2024-11-27T21:35:51,563 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/7036cc8c1caa4629b4097a32c1445360 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/7036cc8c1caa4629b4097a32c1445360 2024-11-27T21:35:51,564 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/7c197dfa054841048cbbc292c592dcf3 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/7c197dfa054841048cbbc292c592dcf3 2024-11-27T21:35:51,565 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/9df5f1a2201f4e7f86f73aeac47d2a42 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/9df5f1a2201f4e7f86f73aeac47d2a42 2024-11-27T21:35:51,566 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/c44dbf6811cd474987c7637b23fb4de0 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/C/c44dbf6811cd474987c7637b23fb4de0 2024-11-27T21:35:51,569 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/recovered.edits/436.seqid to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a/recovered.edits/436.seqid 2024-11-27T21:35:51,570 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/d9a1bb9b05a472888bd945369e4af06a 2024-11-27T21:35:51,570 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-27T21:35:51,571 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=96, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:35:51,575 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-27T21:35:51,577 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-27T21:35:51,578 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=96, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:35:51,578 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-27T21:35:51,579 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732743351578"}]},"ts":"9223372036854775807"} 2024-11-27T21:35:51,580 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-27T21:35:51,580 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d9a1bb9b05a472888bd945369e4af06a, NAME => 'TestAcidGuarantees,,1732743324553.d9a1bb9b05a472888bd945369e4af06a.', STARTKEY => '', ENDKEY => ''}] 2024-11-27T21:35:51,581 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-27T21:35:51,581 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732743351581"}]},"ts":"9223372036854775807"} 2024-11-27T21:35:51,582 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-27T21:35:51,583 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=96, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:35:51,584 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 39 msec 2024-11-27T21:35:51,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-27T21:35:51,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-27T21:35:51,654 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:35:51,662 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=240 (was 242), OpenFileDescriptor=453 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=332 (was 317) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8294 (was 8468) 2024-11-27T21:35:51,671 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=240, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=332, ProcessCount=11, AvailableMemoryMB=8294 2024-11-27T21:35:51,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T21:35:51,673 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T21:35:51,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-27T21:35:51,675 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T21:35:51,675 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:35:51,675 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 97 2024-11-27T21:35:51,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-27T21:35:51,676 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T21:35:51,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742173_1349 (size=959) 2024-11-27T21:35:51,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-27T21:35:51,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-27T21:35:52,084 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d 2024-11-27T21:35:52,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742174_1350 (size=53) 2024-11-27T21:35:52,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-27T21:35:52,489 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T21:35:52,489 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f9b9357dca19fe90b610172c1465ef7c, disabling compactions & flushes 2024-11-27T21:35:52,489 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:35:52,489 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:35:52,489 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. after waiting 0 ms 2024-11-27T21:35:52,489 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:35:52,489 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:35:52,489 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f9b9357dca19fe90b610172c1465ef7c: Waiting for close lock at 1732743352489Disabling compacts and flushes for region at 1732743352489Disabling writes for close at 1732743352489Writing region close event to WAL at 1732743352489Closed at 1732743352489 2024-11-27T21:35:52,490 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T21:35:52,491 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732743352490"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732743352490"}]},"ts":"1732743352490"} 2024-11-27T21:35:52,492 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T21:35:52,493 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T21:35:52,493 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732743352493"}]},"ts":"1732743352493"} 2024-11-27T21:35:52,494 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-27T21:35:52,494 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f9b9357dca19fe90b610172c1465ef7c, ASSIGN}] 2024-11-27T21:35:52,495 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f9b9357dca19fe90b610172c1465ef7c, ASSIGN 2024-11-27T21:35:52,495 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f9b9357dca19fe90b610172c1465ef7c, ASSIGN; state=OFFLINE, location=4585c39bed03,39097,1732743267048; forceNewPlan=false, retain=false 2024-11-27T21:35:52,646 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=f9b9357dca19fe90b610172c1465ef7c, regionState=OPENING, regionLocation=4585c39bed03,39097,1732743267048 2024-11-27T21:35:52,647 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f9b9357dca19fe90b610172c1465ef7c, ASSIGN because future has completed 2024-11-27T21:35:52,648 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=99, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048}] 2024-11-27T21:35:52,802 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:35:52,802 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(7752): Opening region: {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} 2024-11-27T21:35:52,803 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:35:52,803 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T21:35:52,803 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(7794): checking encryption for f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:35:52,803 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(7797): checking classloading for f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:35:52,804 INFO [StoreOpener-f9b9357dca19fe90b610172c1465ef7c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:35:52,805 INFO [StoreOpener-f9b9357dca19fe90b610172c1465ef7c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f9b9357dca19fe90b610172c1465ef7c columnFamilyName A 2024-11-27T21:35:52,805 DEBUG [StoreOpener-f9b9357dca19fe90b610172c1465ef7c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:35:52,806 INFO [StoreOpener-f9b9357dca19fe90b610172c1465ef7c-1 {}] regionserver.HStore(327): Store=f9b9357dca19fe90b610172c1465ef7c/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:35:52,806 INFO [StoreOpener-f9b9357dca19fe90b610172c1465ef7c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:35:52,807 INFO [StoreOpener-f9b9357dca19fe90b610172c1465ef7c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f9b9357dca19fe90b610172c1465ef7c columnFamilyName B 2024-11-27T21:35:52,807 DEBUG [StoreOpener-f9b9357dca19fe90b610172c1465ef7c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:35:52,807 INFO [StoreOpener-f9b9357dca19fe90b610172c1465ef7c-1 {}] regionserver.HStore(327): Store=f9b9357dca19fe90b610172c1465ef7c/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:35:52,807 INFO [StoreOpener-f9b9357dca19fe90b610172c1465ef7c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:35:52,808 INFO [StoreOpener-f9b9357dca19fe90b610172c1465ef7c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f9b9357dca19fe90b610172c1465ef7c columnFamilyName C 2024-11-27T21:35:52,808 DEBUG [StoreOpener-f9b9357dca19fe90b610172c1465ef7c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:35:52,808 INFO [StoreOpener-f9b9357dca19fe90b610172c1465ef7c-1 {}] regionserver.HStore(327): Store=f9b9357dca19fe90b610172c1465ef7c/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:35:52,808 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1038): replaying wal for f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:35:52,809 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:35:52,809 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:35:52,810 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1048): stopping wal replay for f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:35:52,810 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1060): Cleaning up temporary data for f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:35:52,810 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T21:35:52,811 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1093): writing seq id for f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:35:52,812 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T21:35:52,812 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1114): Opened f9b9357dca19fe90b610172c1465ef7c; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60742263, jitterRate=-0.09486974775791168}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T21:35:52,813 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:35:52,813 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1006): Region open journal for f9b9357dca19fe90b610172c1465ef7c: Running coprocessor pre-open hook at 1732743352803Writing region info on filesystem at 1732743352803Initializing all the Stores at 1732743352804 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732743352804Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732743352804Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732743352804Cleaning up temporary data from old regions at 1732743352810 (+6 ms)Running coprocessor post-open hooks at 1732743352813 (+3 ms)Region opened successfully at 1732743352813 2024-11-27T21:35:52,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-27T21:35:52,814 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., pid=99, masterSystemTime=1732743352799 2024-11-27T21:35:52,815 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:35:52,815 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:35:52,815 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=f9b9357dca19fe90b610172c1465ef7c, regionState=OPEN, openSeqNum=2, regionLocation=4585c39bed03,39097,1732743267048 2024-11-27T21:35:52,816 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=99, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 because future has completed 2024-11-27T21:35:52,818 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=99, resume processing ppid=98 2024-11-27T21:35:52,819 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, ppid=98, state=SUCCESS, hasLock=false; OpenRegionProcedure f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 in 169 msec 2024-11-27T21:35:52,820 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-27T21:35:52,820 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f9b9357dca19fe90b610172c1465ef7c, ASSIGN in 325 msec 2024-11-27T21:35:52,820 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T21:35:52,821 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732743352820"}]},"ts":"1732743352820"} 2024-11-27T21:35:52,822 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-27T21:35:52,822 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T21:35:52,824 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1490 sec 2024-11-27T21:35:53,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-27T21:35:53,823 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:35:53,824 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d493646, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:53,824 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:35:53,825 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:35:53,826 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:35:53,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:35:53,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:35:53,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25150641, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:53,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:35:53,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:35:53,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:53,827 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41846, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:35:53,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6af2ce89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:53,828 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:35:53,829 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:35:53,829 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:35:53,830 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:35:53,830 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T21:35:53,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 4585c39bed03,34433,1732743266181 2024-11-27T21:35:53,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@715115ab 2024-11-27T21:35:53,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T21:35:53,832 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41852, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T21:35:53,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T21:35:53,833 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T21:35:53,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=100, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-27T21:35:53,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742175_1351 (size=995) 2024-11-27T21:35:54,244 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-27T21:35:54,244 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-27T21:35:54,246 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=101, ppid=100, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T21:35:54,248 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f9b9357dca19fe90b610172c1465ef7c, REOPEN/MOVE}] 2024-11-27T21:35:54,249 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f9b9357dca19fe90b610172c1465ef7c, REOPEN/MOVE 2024-11-27T21:35:54,249 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=f9b9357dca19fe90b610172c1465ef7c, regionState=CLOSING, regionLocation=4585c39bed03,39097,1732743267048 2024-11-27T21:35:54,251 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f9b9357dca19fe90b610172c1465ef7c, REOPEN/MOVE because future has completed 2024-11-27T21:35:54,251 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T21:35:54,251 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=103, ppid=102, state=RUNNABLE, hasLock=false; CloseRegionProcedure f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048}] 2024-11-27T21:35:54,403 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] handler.UnassignRegionHandler(122): Close f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:35:54,403 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T21:35:54,403 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1722): Closing f9b9357dca19fe90b610172c1465ef7c, disabling compactions & flushes 2024-11-27T21:35:54,403 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:35:54,403 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:35:54,403 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. after waiting 0 ms 2024-11-27T21:35:54,403 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:35:54,406 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T21:35:54,407 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:35:54,407 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1676): Region close journal for f9b9357dca19fe90b610172c1465ef7c: Waiting for close lock at 1732743354403Running coprocessor pre-close hooks at 1732743354403Disabling compacts and flushes for region at 1732743354403Disabling writes for close at 1732743354403Writing region close event to WAL at 1732743354403Running coprocessor post-close hooks at 1732743354407 (+4 ms)Closed at 1732743354407 2024-11-27T21:35:54,407 WARN [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegionServer(3299): Not adding moved region record: f9b9357dca19fe90b610172c1465ef7c to self. 2024-11-27T21:35:54,408 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] handler.UnassignRegionHandler(157): Closed f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:35:54,409 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=f9b9357dca19fe90b610172c1465ef7c, regionState=CLOSED 2024-11-27T21:35:54,410 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=103, ppid=102, state=RUNNABLE, hasLock=false; CloseRegionProcedure f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 because future has completed 2024-11-27T21:35:54,411 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=103, resume processing ppid=102 2024-11-27T21:35:54,412 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, ppid=102, state=SUCCESS, hasLock=false; CloseRegionProcedure f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 in 159 msec 2024-11-27T21:35:54,412 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f9b9357dca19fe90b610172c1465ef7c, REOPEN/MOVE; state=CLOSED, location=4585c39bed03,39097,1732743267048; forceNewPlan=false, retain=true 2024-11-27T21:35:54,563 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=f9b9357dca19fe90b610172c1465ef7c, regionState=OPENING, regionLocation=4585c39bed03,39097,1732743267048 2024-11-27T21:35:54,564 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f9b9357dca19fe90b610172c1465ef7c, REOPEN/MOVE because future has completed 2024-11-27T21:35:54,564 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=102, state=RUNNABLE, hasLock=false; OpenRegionProcedure f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048}] 2024-11-27T21:35:54,718 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:35:54,719 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(7752): Opening region: {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} 2024-11-27T21:35:54,719 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:35:54,719 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T21:35:54,719 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(7794): checking encryption for f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:35:54,719 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(7797): checking classloading for f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:35:54,720 INFO [StoreOpener-f9b9357dca19fe90b610172c1465ef7c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:35:54,721 INFO [StoreOpener-f9b9357dca19fe90b610172c1465ef7c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f9b9357dca19fe90b610172c1465ef7c columnFamilyName A 2024-11-27T21:35:54,722 DEBUG [StoreOpener-f9b9357dca19fe90b610172c1465ef7c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:35:54,722 INFO [StoreOpener-f9b9357dca19fe90b610172c1465ef7c-1 {}] regionserver.HStore(327): Store=f9b9357dca19fe90b610172c1465ef7c/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:35:54,723 INFO [StoreOpener-f9b9357dca19fe90b610172c1465ef7c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:35:54,723 INFO [StoreOpener-f9b9357dca19fe90b610172c1465ef7c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f9b9357dca19fe90b610172c1465ef7c columnFamilyName B 2024-11-27T21:35:54,723 DEBUG [StoreOpener-f9b9357dca19fe90b610172c1465ef7c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:35:54,724 INFO [StoreOpener-f9b9357dca19fe90b610172c1465ef7c-1 {}] regionserver.HStore(327): Store=f9b9357dca19fe90b610172c1465ef7c/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:35:54,724 INFO [StoreOpener-f9b9357dca19fe90b610172c1465ef7c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:35:54,724 INFO [StoreOpener-f9b9357dca19fe90b610172c1465ef7c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f9b9357dca19fe90b610172c1465ef7c columnFamilyName C 2024-11-27T21:35:54,724 DEBUG [StoreOpener-f9b9357dca19fe90b610172c1465ef7c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:35:54,725 INFO [StoreOpener-f9b9357dca19fe90b610172c1465ef7c-1 {}] regionserver.HStore(327): Store=f9b9357dca19fe90b610172c1465ef7c/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:35:54,725 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1038): replaying wal for f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:35:54,725 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:35:54,726 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:35:54,727 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1048): stopping wal replay for f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:35:54,727 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1060): Cleaning up temporary data for f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:35:54,727 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T21:35:54,728 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1093): writing seq id for f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:35:54,729 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1114): Opened f9b9357dca19fe90b610172c1465ef7c; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60682648, jitterRate=-0.09575808048248291}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T21:35:54,729 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:35:54,729 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1006): Region open journal for f9b9357dca19fe90b610172c1465ef7c: Running coprocessor pre-open hook at 1732743354719Writing region info on filesystem at 1732743354719Initializing all the Stores at 1732743354720 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732743354720Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732743354720Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732743354720Cleaning up temporary data from old regions at 1732743354727 (+7 ms)Running coprocessor post-open hooks at 1732743354729 (+2 ms)Region opened successfully at 1732743354729 2024-11-27T21:35:54,730 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., pid=104, masterSystemTime=1732743354716 2024-11-27T21:35:54,732 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:35:54,732 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:35:54,732 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=f9b9357dca19fe90b610172c1465ef7c, regionState=OPEN, openSeqNum=5, regionLocation=4585c39bed03,39097,1732743267048 2024-11-27T21:35:54,733 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=104, ppid=102, state=RUNNABLE, hasLock=false; OpenRegionProcedure f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 because future has completed 2024-11-27T21:35:54,735 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=102 2024-11-27T21:35:54,735 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=102, state=SUCCESS, hasLock=false; OpenRegionProcedure f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 in 170 msec 2024-11-27T21:35:54,737 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-27T21:35:54,737 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f9b9357dca19fe90b610172c1465ef7c, REOPEN/MOVE in 487 msec 2024-11-27T21:35:54,739 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=101, resume processing ppid=100 2024-11-27T21:35:54,739 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, ppid=100, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 491 msec 2024-11-27T21:35:54,741 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 905 msec 2024-11-27T21:35:54,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=100 2024-11-27T21:35:54,741 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:35:54,742 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c6bde19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:54,742 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:35:54,742 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:35:54,743 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:35:54,743 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:35:54,743 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:35:54,743 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3094ef98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:54,743 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:35:54,743 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:35:54,744 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:54,744 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41870, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:35:54,745 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46a955dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:54,745 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@173aa1ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:54,745 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:35:54,745 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:35:54,746 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:35:54,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:35:54,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:35:54,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f59e88f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:54,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:35:54,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:35:54,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:54,747 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41876, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:35:54,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@585cbc91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:54,748 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d5e8400, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:54,748 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:35:54,748 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:35:54,749 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:35:54,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:35:54,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:35:54,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5109cac1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:54,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:35:54,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:35:54,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:54,750 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41900, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:35:54,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a06f1c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:54,751 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ce5c6bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:54,751 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:35:54,751 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:35:54,752 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:35:54,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:35:54,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:35:54,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fb5d934, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:54,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:35:54,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:35:54,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:54,753 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41928, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:35:54,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@210aa1c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:54,754 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b7fbdbe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:54,754 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:35:54,754 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:35:54,754 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:35:54,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:35:54,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:35:54,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d8357f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:54,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:35:54,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:35:54,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:54,755 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41946, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:35:54,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38278641, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:54,756 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69edc398, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:54,756 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:35:54,756 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:35:54,757 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:35:54,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:35:54,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:35:54,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f286c94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:54,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:35:54,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:35:54,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:54,758 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:35:54,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@645cf045, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:54,759 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bbf44a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:54,759 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:35:54,759 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:35:54,759 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:35:54,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:35:54,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:35:54,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bc7fd42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:54,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:35:54,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:35:54,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:54,761 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:35:54,761 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cb163ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:54,761 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9ad32ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:54,761 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:35:54,762 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:35:54,762 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:35:54,762 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:35:54,762 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:35:54,762 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6453f94f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:54,762 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:35:54,763 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:35:54,763 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:54,763 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:35:54,764 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3420b3a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:54,764 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@615de503, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:54,764 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:35:54,764 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:35:54,765 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:35:54,765 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:35:54,765 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:35:54,765 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@141971d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:54,765 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:35:54,765 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:35:54,765 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:54,766 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42040, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:35:54,766 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5070863, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:54,767 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5690e3aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:54,767 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:35:54,767 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:35:54,767 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:35:54,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:35:54,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:35:54,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38c61ed6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:54,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:35:54,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:35:54,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:35:54,769 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42064, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:35:54,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@440da3ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:35:54,770 DEBUG [Thread-1607 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:35:54,770 DEBUG [Thread-1603 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:35:54,770 DEBUG [Thread-1601 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:35:54,770 DEBUG [Thread-1605 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:35:54,770 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:35:54,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:35:54,770 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:35:54,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:35:54,771 DEBUG [Thread-1603 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:35:54,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:35:54,771 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:35:54,771 DEBUG [Thread-1603 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:35:54,772 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42748, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:35:54,772 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42744, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:35:54,772 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42750, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:35:54,772 DEBUG [Thread-1592 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:35:54,772 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42754, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:35:54,772 DEBUG [Thread-1594 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:35:54,772 DEBUG [Thread-1596 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:35:54,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:35:54,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:35:54,773 DEBUG [Thread-1590 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:35:54,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:35:54,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:35:54,773 DEBUG [Thread-1588 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:35:54,774 DEBUG [Thread-1599 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:35:54,774 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:35:54,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:35:54,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:35:54,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:35:54,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:35:54,775 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:35:54,775 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:35:54,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5] 2024-11-27T21:35:54,775 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5] 2024-11-27T21:35:54,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:35:54,776 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42762, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:35:54,776 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42776, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:35:54,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:35:54,776 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42784, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:35:54,776 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42806, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:35:54,776 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42800, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:35:54,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5] 2024-11-27T21:35:54,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-27T21:35:54,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5] 2024-11-27T21:35:54,777 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42814, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:35:54,778 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:35:54,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-27T21:35:54,778 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:35:54,779 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:35:54,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5] 2024-11-27T21:35:54,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5] 2024-11-27T21:35:54,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5] 2024-11-27T21:35:54,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5] 2024-11-27T21:35:54,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5] 2024-11-27T21:35:54,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5] 2024-11-27T21:35:54,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:35:54,789 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:35:54,789 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f9b9357dca19fe90b610172c1465ef7c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T21:35:54,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411274029b6863c51432098449af8097c4dc5_f9b9357dca19fe90b610172c1465ef7c is 50, key is test_row_0/A:col10/1732743354785/Put/seqid=0 2024-11-27T21:35:54,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742176_1352 (size=12154) 2024-11-27T21:35:54,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:54,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743414836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:54,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:54,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:54,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:54,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:54,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743414838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:54,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:54,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743414839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:54,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:54,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:54,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:54,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:54,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:54,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:54,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:54,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743414839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:54,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:54,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:54,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:54,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:54,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743414840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:54,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:54,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:54,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:54,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-27T21:35:54,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-27T21:35:54,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:35:54,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. as already flushing 2024-11-27T21:35:54,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:35:54,934 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:54,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:54,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:54,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743414954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:54,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:54,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:54,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743414954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:54,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:54,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:54,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:54,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743414954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:54,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:54,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:54,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:54,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:54,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743414954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:54,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:54,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:54,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:54,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743414954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:54,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:54,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:54,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:55,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-27T21:35:55,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:35:55,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. as already flushing 2024-11-27T21:35:55,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:35:55,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:55,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:55,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:55,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-27T21:35:55,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:55,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743415173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:55,178 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:55,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:55,178 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:55,178 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:55,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743415173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:55,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:55,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:55,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:55,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743415173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:55,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:55,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:55,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:55,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:55,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:55,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743415173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:55,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:55,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:55,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:55,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:55,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743415174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:55,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:55,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:55,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:55,198 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T21:35:55,212 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T21:35:55,239 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:35:55,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-27T21:35:55,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:35:55,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. as already flushing 2024-11-27T21:35:55,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:35:55,240 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:55,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:55,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:55,243 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411274029b6863c51432098449af8097c4dc5_f9b9357dca19fe90b610172c1465ef7c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411274029b6863c51432098449af8097c4dc5_f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:35:55,244 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/21027248c3d74c79963bc7530b9ef8f5, store: [table=TestAcidGuarantees family=A region=f9b9357dca19fe90b610172c1465ef7c] 2024-11-27T21:35:55,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/21027248c3d74c79963bc7530b9ef8f5 is 175, key is test_row_0/A:col10/1732743354785/Put/seqid=0 2024-11-27T21:35:55,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742177_1353 (size=30955) 2024-11-27T21:35:55,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-27T21:35:55,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:35:55,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. as already flushing 2024-11-27T21:35:55,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:35:55,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:55,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:55,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:55,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-27T21:35:55,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:55,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743415483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:55,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:55,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:55,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743415483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:55,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:55,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:55,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:55,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:55,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:55,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743415483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:55,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:55,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:55,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:55,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:55,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743415483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:55,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:55,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:55,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:55,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743415483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:55,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:55,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:55,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:55,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:55,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:55,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-27T21:35:55,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:35:55,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. as already flushing 2024-11-27T21:35:55,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:35:55,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:55,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:55,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:55,649 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/21027248c3d74c79963bc7530b9ef8f5 2024-11-27T21:35:55,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/aa4d407982d842fcbec4b0994ee9936a is 50, key is test_row_0/B:col10/1732743354785/Put/seqid=0 2024-11-27T21:35:55,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742178_1354 (size=12001) 2024-11-27T21:35:55,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-27T21:35:55,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:35:55,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. as already flushing 2024-11-27T21:35:55,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:35:55,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:55,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:55,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:55,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-27T21:35:55,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:35:55,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. as already flushing 2024-11-27T21:35:55,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:35:55,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:55,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:55,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:55,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-27T21:35:55,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:55,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743415993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:55,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:55,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:55,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:55,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:55,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743415993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:55,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:55,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:55,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:55,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743415993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:55,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:55,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:55,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:56,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-27T21:35:56,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:35:56,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. as already flushing 2024-11-27T21:35:56,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:35:56,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:56,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:56,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:35:56,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:56,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743416003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:56,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:56,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:56,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:56,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743416003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:56,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:56,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:56,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:56,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:56,081 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/aa4d407982d842fcbec4b0994ee9936a 2024-11-27T21:35:56,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/8bc59e85e42644599830c06e6a62d6e4 is 50, key is test_row_0/C:col10/1732743354785/Put/seqid=0 2024-11-27T21:35:56,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742179_1355 (size=12001) 2024-11-27T21:35:56,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/8bc59e85e42644599830c06e6a62d6e4 2024-11-27T21:35:56,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/21027248c3d74c79963bc7530b9ef8f5 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/21027248c3d74c79963bc7530b9ef8f5 2024-11-27T21:35:56,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/21027248c3d74c79963bc7530b9ef8f5, entries=150, sequenceid=15, filesize=30.2 K 2024-11-27T21:35:56,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/aa4d407982d842fcbec4b0994ee9936a as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/aa4d407982d842fcbec4b0994ee9936a 2024-11-27T21:35:56,120 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/aa4d407982d842fcbec4b0994ee9936a, entries=150, sequenceid=15, filesize=11.7 K 2024-11-27T21:35:56,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/8bc59e85e42644599830c06e6a62d6e4 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/8bc59e85e42644599830c06e6a62d6e4 2024-11-27T21:35:56,126 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/8bc59e85e42644599830c06e6a62d6e4, entries=150, sequenceid=15, filesize=11.7 K 2024-11-27T21:35:56,127 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for f9b9357dca19fe90b610172c1465ef7c in 1337ms, sequenceid=15, compaction requested=false 2024-11-27T21:35:56,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f9b9357dca19fe90b610172c1465ef7c: 2024-11-27T21:35:56,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-27T21:35:56,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:35:56,159 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing f9b9357dca19fe90b610172c1465ef7c 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T21:35:56,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411278f26c69e7f624d45bda13b3e0f3290c6_f9b9357dca19fe90b610172c1465ef7c is 50, key is test_row_0/A:col10/1732743354835/Put/seqid=0 2024-11-27T21:35:56,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742180_1356 (size=12154) 2024-11-27T21:35:56,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:35:56,578 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411278f26c69e7f624d45bda13b3e0f3290c6_f9b9357dca19fe90b610172c1465ef7c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411278f26c69e7f624d45bda13b3e0f3290c6_f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:35:56,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/0c26a6577f804813a2d786a100af8e7d, store: [table=TestAcidGuarantees family=A region=f9b9357dca19fe90b610172c1465ef7c] 2024-11-27T21:35:56,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/0c26a6577f804813a2d786a100af8e7d is 175, key is test_row_0/A:col10/1732743354835/Put/seqid=0 2024-11-27T21:35:56,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742181_1357 (size=30955) 2024-11-27T21:35:56,585 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/0c26a6577f804813a2d786a100af8e7d 2024-11-27T21:35:56,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/21703d718246482a880638d3a7718c47 is 50, key is test_row_0/B:col10/1732743354835/Put/seqid=0 2024-11-27T21:35:56,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742182_1358 (size=12001) 2024-11-27T21:35:56,611 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/21703d718246482a880638d3a7718c47 2024-11-27T21:35:56,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/1e6432ce99864782af8c183a910a8071 is 50, key is test_row_0/C:col10/1732743354835/Put/seqid=0 2024-11-27T21:35:56,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742183_1359 (size=12001) 2024-11-27T21:35:56,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-27T21:35:57,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:35:57,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. as already flushing 2024-11-27T21:35:57,030 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/1e6432ce99864782af8c183a910a8071 2024-11-27T21:35:57,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/0c26a6577f804813a2d786a100af8e7d as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/0c26a6577f804813a2d786a100af8e7d 2024-11-27T21:35:57,037 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/0c26a6577f804813a2d786a100af8e7d, entries=150, sequenceid=40, filesize=30.2 K 2024-11-27T21:35:57,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/21703d718246482a880638d3a7718c47 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/21703d718246482a880638d3a7718c47 2024-11-27T21:35:57,041 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/21703d718246482a880638d3a7718c47, entries=150, sequenceid=40, filesize=11.7 K 2024-11-27T21:35:57,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/1e6432ce99864782af8c183a910a8071 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/1e6432ce99864782af8c183a910a8071 2024-11-27T21:35:57,044 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/1e6432ce99864782af8c183a910a8071, entries=150, sequenceid=40, filesize=11.7 K 2024-11-27T21:35:57,051 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=20.13 KB/20610 for f9b9357dca19fe90b610172c1465ef7c in 892ms, sequenceid=40, compaction requested=false 2024-11-27T21:35:57,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for f9b9357dca19fe90b610172c1465ef7c: 2024-11-27T21:35:57,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:35:57,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-27T21:35:57,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-27T21:35:57,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:35:57,053 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:35:57,053 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f9b9357dca19fe90b610172c1465ef7c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T21:35:57,054 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-27T21:35:57,054 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2730 sec 2024-11-27T21:35:57,056 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.2810 sec 2024-11-27T21:35:57,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127fa2b8d83e4bb45d6af05f18b6dad7a45_f9b9357dca19fe90b610172c1465ef7c is 50, key is test_row_0/A:col10/1732743357052/Put/seqid=0 2024-11-27T21:35:57,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742184_1360 (size=17034) 2024-11-27T21:35:57,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:57,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743417105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:57,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:57,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:57,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743417105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:57,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:57,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:57,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:57,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:57,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:57,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:57,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743417107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:57,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:57,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:57,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:57,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:57,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743417107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:57,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:57,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:57,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:57,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:57,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743417112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:57,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:57,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:57,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:57,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:57,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743417223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:57,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:57,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:57,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:57,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743417223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:57,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:57,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:57,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743417223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:57,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:57,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:57,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:57,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:57,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:57,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:57,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743417223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:57,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:57,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:57,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:57,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:57,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:57,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743417233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:57,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:57,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:57,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:57,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:57,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743417443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:57,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:57,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:57,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743417444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:57,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:57,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:57,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:57,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:57,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:57,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743417444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:57,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:57,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:57,454 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:57,454 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:57,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743417444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:57,454 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:57,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:57,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:57,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:57,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:57,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743417444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:57,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:57,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:57,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:57,465 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:35:57,468 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127fa2b8d83e4bb45d6af05f18b6dad7a45_f9b9357dca19fe90b610172c1465ef7c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127fa2b8d83e4bb45d6af05f18b6dad7a45_f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:35:57,469 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/cffcaa3e5adf4679af73ee39e56feac8, store: [table=TestAcidGuarantees family=A region=f9b9357dca19fe90b610172c1465ef7c] 2024-11-27T21:35:57,469 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/cffcaa3e5adf4679af73ee39e56feac8 is 175, key is test_row_0/A:col10/1732743357052/Put/seqid=0 2024-11-27T21:35:57,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742185_1361 (size=48139) 2024-11-27T21:35:57,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:57,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743417763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:57,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:57,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:57,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:57,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:57,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743417763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:57,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:57,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743417763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:57,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:57,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:57,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:57,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:57,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:57,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:57,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:57,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743417763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:57,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:57,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:57,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:57,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:57,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743417764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:57,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:57,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:57,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:57,874 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/cffcaa3e5adf4679af73ee39e56feac8 2024-11-27T21:35:57,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/44c573df0b984de68778e636f8e1ab81 is 50, key is test_row_0/B:col10/1732743357052/Put/seqid=0 2024-11-27T21:35:57,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742186_1362 (size=12001) 2024-11-27T21:35:58,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:58,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743418273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:58,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:58,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:58,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:58,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:58,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743418273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:58,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:58,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:58,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:58,284 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/44c573df0b984de68778e636f8e1ab81 2024-11-27T21:35:58,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/061e32d4666c4b00ba58102dcca54e15 is 50, key is test_row_0/C:col10/1732743357052/Put/seqid=0 2024-11-27T21:35:58,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742187_1363 (size=12001) 2024-11-27T21:35:58,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:58,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743418284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:58,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:58,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:58,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:58,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:58,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743418284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:58,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:58,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:58,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:58,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:58,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743418284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:58,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:58,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:58,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:58,694 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/061e32d4666c4b00ba58102dcca54e15 2024-11-27T21:35:58,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/cffcaa3e5adf4679af73ee39e56feac8 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/cffcaa3e5adf4679af73ee39e56feac8 2024-11-27T21:35:58,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/cffcaa3e5adf4679af73ee39e56feac8, entries=250, sequenceid=51, filesize=47.0 K 2024-11-27T21:35:58,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/44c573df0b984de68778e636f8e1ab81 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/44c573df0b984de68778e636f8e1ab81 2024-11-27T21:35:58,706 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/44c573df0b984de68778e636f8e1ab81, entries=150, sequenceid=51, filesize=11.7 K 2024-11-27T21:35:58,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/061e32d4666c4b00ba58102dcca54e15 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/061e32d4666c4b00ba58102dcca54e15 2024-11-27T21:35:58,710 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/061e32d4666c4b00ba58102dcca54e15, entries=150, sequenceid=51, filesize=11.7 K 2024-11-27T21:35:58,711 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for f9b9357dca19fe90b610172c1465ef7c in 1658ms, sequenceid=51, compaction requested=true 2024-11-27T21:35:58,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f9b9357dca19fe90b610172c1465ef7c: 2024-11-27T21:35:58,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f9b9357dca19fe90b610172c1465ef7c:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:35:58,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:58,711 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:35:58,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f9b9357dca19fe90b610172c1465ef7c:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:35:58,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:58,712 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:35:58,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f9b9357dca19fe90b610172c1465ef7c:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:35:58,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:35:58,712 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:35:58,712 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): f9b9357dca19fe90b610172c1465ef7c/A is initiating minor compaction (all files) 2024-11-27T21:35:58,712 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f9b9357dca19fe90b610172c1465ef7c/A in TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:35:58,713 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/21027248c3d74c79963bc7530b9ef8f5, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/0c26a6577f804813a2d786a100af8e7d, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/cffcaa3e5adf4679af73ee39e56feac8] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp, totalSize=107.5 K 2024-11-27T21:35:58,713 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:35:58,713 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. files: [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/21027248c3d74c79963bc7530b9ef8f5, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/0c26a6577f804813a2d786a100af8e7d, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/cffcaa3e5adf4679af73ee39e56feac8] 2024-11-27T21:35:58,713 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:35:58,713 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): f9b9357dca19fe90b610172c1465ef7c/B is initiating minor compaction (all files) 2024-11-27T21:35:58,713 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21027248c3d74c79963bc7530b9ef8f5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732743354785 2024-11-27T21:35:58,713 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f9b9357dca19fe90b610172c1465ef7c/B in TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:35:58,713 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/aa4d407982d842fcbec4b0994ee9936a, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/21703d718246482a880638d3a7718c47, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/44c573df0b984de68778e636f8e1ab81] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp, totalSize=35.2 K 2024-11-27T21:35:58,713 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c26a6577f804813a2d786a100af8e7d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732743354835 2024-11-27T21:35:58,713 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting aa4d407982d842fcbec4b0994ee9936a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732743354785 2024-11-27T21:35:58,714 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting cffcaa3e5adf4679af73ee39e56feac8, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732743357017 2024-11-27T21:35:58,714 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 21703d718246482a880638d3a7718c47, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732743354835 2024-11-27T21:35:58,714 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 44c573df0b984de68778e636f8e1ab81, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732743357022 2024-11-27T21:35:58,719 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f9b9357dca19fe90b610172c1465ef7c] 2024-11-27T21:35:58,721 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f9b9357dca19fe90b610172c1465ef7c#B#compaction#314 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:58,721 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127a9dd1b2d076b437895dea5124971466a_f9b9357dca19fe90b610172c1465ef7c store=[table=TestAcidGuarantees family=A region=f9b9357dca19fe90b610172c1465ef7c] 2024-11-27T21:35:58,721 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/e283383e8a39481ba63bc1ea8b900cb5 is 50, key is test_row_0/B:col10/1732743357052/Put/seqid=0 2024-11-27T21:35:58,723 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127a9dd1b2d076b437895dea5124971466a_f9b9357dca19fe90b610172c1465ef7c, store=[table=TestAcidGuarantees family=A region=f9b9357dca19fe90b610172c1465ef7c] 2024-11-27T21:35:58,723 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127a9dd1b2d076b437895dea5124971466a_f9b9357dca19fe90b610172c1465ef7c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f9b9357dca19fe90b610172c1465ef7c] 2024-11-27T21:35:58,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742188_1364 (size=12104) 2024-11-27T21:35:58,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742189_1365 (size=4469) 2024-11-27T21:35:58,740 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/e283383e8a39481ba63bc1ea8b900cb5 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/e283383e8a39481ba63bc1ea8b900cb5 2024-11-27T21:35:58,746 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f9b9357dca19fe90b610172c1465ef7c/B of f9b9357dca19fe90b610172c1465ef7c into e283383e8a39481ba63bc1ea8b900cb5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:58,746 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f9b9357dca19fe90b610172c1465ef7c: 2024-11-27T21:35:58,746 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., storeName=f9b9357dca19fe90b610172c1465ef7c/B, priority=13, startTime=1732743358711; duration=0sec 2024-11-27T21:35:58,746 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:35:58,746 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f9b9357dca19fe90b610172c1465ef7c:B 2024-11-27T21:35:58,746 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:35:58,747 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:35:58,747 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): f9b9357dca19fe90b610172c1465ef7c/C is initiating minor compaction (all files) 2024-11-27T21:35:58,747 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f9b9357dca19fe90b610172c1465ef7c/C in TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:35:58,747 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/8bc59e85e42644599830c06e6a62d6e4, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/1e6432ce99864782af8c183a910a8071, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/061e32d4666c4b00ba58102dcca54e15] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp, totalSize=35.2 K 2024-11-27T21:35:58,748 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bc59e85e42644599830c06e6a62d6e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732743354785 2024-11-27T21:35:58,748 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e6432ce99864782af8c183a910a8071, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732743354835 2024-11-27T21:35:58,749 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 061e32d4666c4b00ba58102dcca54e15, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732743357022 2024-11-27T21:35:58,755 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f9b9357dca19fe90b610172c1465ef7c#C#compaction#315 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:58,755 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/bc050f7cbf2e4c83867bd8acad07e40d is 50, key is test_row_0/C:col10/1732743357052/Put/seqid=0 2024-11-27T21:35:58,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742190_1366 (size=12104) 2024-11-27T21:35:58,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-27T21:35:58,944 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:35:58,944 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:35:58,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-27T21:35:58,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-27T21:35:58,946 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:35:58,947 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:35:58,947 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:35:59,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-27T21:35:59,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-27T21:35:59,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:35:59,099 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing f9b9357dca19fe90b610172c1465ef7c 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T21:35:59,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411274b9ea8f6e0fe4f4ab7ea70ad9916e426_f9b9357dca19fe90b610172c1465ef7c is 50, key is test_row_0/A:col10/1732743357103/Put/seqid=0 2024-11-27T21:35:59,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742191_1367 (size=12154) 2024-11-27T21:35:59,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:35:59,119 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411274b9ea8f6e0fe4f4ab7ea70ad9916e426_f9b9357dca19fe90b610172c1465ef7c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411274b9ea8f6e0fe4f4ab7ea70ad9916e426_f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:35:59,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/01c0c77496764d46a126dd00f57db203, store: [table=TestAcidGuarantees family=A region=f9b9357dca19fe90b610172c1465ef7c] 2024-11-27T21:35:59,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/01c0c77496764d46a126dd00f57db203 is 175, key is test_row_0/A:col10/1732743357103/Put/seqid=0 2024-11-27T21:35:59,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742192_1368 (size=30955) 2024-11-27T21:35:59,133 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/01c0c77496764d46a126dd00f57db203 2024-11-27T21:35:59,136 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f9b9357dca19fe90b610172c1465ef7c#A#compaction#313 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:35:59,136 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/9954eb11b40046679ba847a6d96504ab is 175, key is test_row_0/A:col10/1732743357052/Put/seqid=0 2024-11-27T21:35:59,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/c7f4756b6f764330bcf0a00cf5ac0a04 is 50, key is test_row_0/B:col10/1732743357103/Put/seqid=0 2024-11-27T21:35:59,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742193_1369 (size=31058) 2024-11-27T21:35:59,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742194_1370 (size=12001) 2024-11-27T21:35:59,158 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/c7f4756b6f764330bcf0a00cf5ac0a04 2024-11-27T21:35:59,165 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/bc050f7cbf2e4c83867bd8acad07e40d as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/bc050f7cbf2e4c83867bd8acad07e40d 2024-11-27T21:35:59,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/0d14d18731b0430facbaf7e0d6525c8e is 50, key is test_row_0/C:col10/1732743357103/Put/seqid=0 2024-11-27T21:35:59,171 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f9b9357dca19fe90b610172c1465ef7c/C of f9b9357dca19fe90b610172c1465ef7c into bc050f7cbf2e4c83867bd8acad07e40d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:59,171 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f9b9357dca19fe90b610172c1465ef7c: 2024-11-27T21:35:59,171 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., storeName=f9b9357dca19fe90b610172c1465ef7c/C, priority=13, startTime=1732743358712; duration=0sec 2024-11-27T21:35:59,171 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:59,171 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f9b9357dca19fe90b610172c1465ef7c:C 2024-11-27T21:35:59,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742195_1371 (size=12001) 2024-11-27T21:35:59,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-27T21:35:59,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:35:59,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. as already flushing 2024-11-27T21:35:59,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:59,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743419311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:59,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:59,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:59,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:59,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:59,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743419311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:59,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:59,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:59,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:59,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:59,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743419312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:59,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:59,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743419313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:59,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:59,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:59,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:59,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:59,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:59,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743419313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:59,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:59,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:59,322 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,39097,1732743267048 for 'test_row_0' in f9b9357dca19fe90b610172c1465ef7c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:35:59,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:59,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:59,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:59,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:59,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743419423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:59,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:59,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743419423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:59,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:59,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:59,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:59,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:59,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:59,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743419423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:59,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:59,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:59,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:59,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:59,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:59,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:59,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743419433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:59,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:59,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:59,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:59,552 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/9954eb11b40046679ba847a6d96504ab as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/9954eb11b40046679ba847a6d96504ab 2024-11-27T21:35:59,556 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f9b9357dca19fe90b610172c1465ef7c/A of f9b9357dca19fe90b610172c1465ef7c into 9954eb11b40046679ba847a6d96504ab(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:35:59,556 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f9b9357dca19fe90b610172c1465ef7c: 2024-11-27T21:35:59,556 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., storeName=f9b9357dca19fe90b610172c1465ef7c/A, priority=13, startTime=1732743358711; duration=0sec 2024-11-27T21:35:59,556 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:35:59,556 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f9b9357dca19fe90b610172c1465ef7c:A 2024-11-27T21:35:59,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-27T21:35:59,599 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/0d14d18731b0430facbaf7e0d6525c8e 2024-11-27T21:35:59,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/01c0c77496764d46a126dd00f57db203 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/01c0c77496764d46a126dd00f57db203 2024-11-27T21:35:59,606 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/01c0c77496764d46a126dd00f57db203, entries=150, sequenceid=77, filesize=30.2 K 2024-11-27T21:35:59,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/c7f4756b6f764330bcf0a00cf5ac0a04 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/c7f4756b6f764330bcf0a00cf5ac0a04 2024-11-27T21:35:59,611 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/c7f4756b6f764330bcf0a00cf5ac0a04, entries=150, sequenceid=77, filesize=11.7 K 2024-11-27T21:35:59,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/0d14d18731b0430facbaf7e0d6525c8e as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/0d14d18731b0430facbaf7e0d6525c8e 2024-11-27T21:35:59,615 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/0d14d18731b0430facbaf7e0d6525c8e, entries=150, sequenceid=77, filesize=11.7 K 2024-11-27T21:35:59,616 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for f9b9357dca19fe90b610172c1465ef7c in 517ms, sequenceid=77, compaction requested=false 2024-11-27T21:35:59,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for f9b9357dca19fe90b610172c1465ef7c: 2024-11-27T21:35:59,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:35:59,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-27T21:35:59,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-27T21:35:59,620 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-27T21:35:59,620 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 671 msec 2024-11-27T21:35:59,621 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 676 msec 2024-11-27T21:35:59,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:35:59,636 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:35:59,636 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f9b9357dca19fe90b610172c1465ef7c 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-27T21:35:59,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d3785cec459245329abe66d68a8302f8_f9b9357dca19fe90b610172c1465ef7c is 50, key is test_row_0/A:col10/1732743359309/Put/seqid=0 2024-11-27T21:35:59,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742196_1372 (size=17034) 2024-11-27T21:35:59,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:59,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743419685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:59,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:59,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:59,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743419686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:59,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:59,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:59,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:59,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:59,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:59,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:59,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743419692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:59,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:59,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:59,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:59,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:59,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743419694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:59,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:59,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:59,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:59,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:59,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743419803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:59,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:59,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:59,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:59,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743419803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:59,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:59,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:59,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:59,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:59,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:59,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743419813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:59,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:35:59,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:59,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743419813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:35:59,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:59,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:35:59,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:59,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:35:59,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:00,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:00,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743420013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:00,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:00,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:00,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:00,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743420013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:00,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:00,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:00,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:00,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:00,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:00,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743420023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:00,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:00,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:00,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:00,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743420023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:00,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:00,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:00,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:00,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:00,056 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:36:00,060 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d3785cec459245329abe66d68a8302f8_f9b9357dca19fe90b610172c1465ef7c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d3785cec459245329abe66d68a8302f8_f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:36:00,060 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/e9f2aed1d7d94d2884265b9ee63e79f5, store: [table=TestAcidGuarantees family=A region=f9b9357dca19fe90b610172c1465ef7c] 2024-11-27T21:36:00,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/e9f2aed1d7d94d2884265b9ee63e79f5 is 175, key is test_row_0/A:col10/1732743359309/Put/seqid=0 2024-11-27T21:36:00,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742197_1373 (size=48139) 2024-11-27T21:36:00,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-27T21:36:00,083 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:36:00,083 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:36:00,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-27T21:36:00,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-27T21:36:00,085 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:36:00,086 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:36:00,086 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:36:00,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-27T21:36:00,222 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T21:36:00,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-27T21:36:00,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:00,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. as already flushing 2024-11-27T21:36:00,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:00,238 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:00,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:00,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:00,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:00,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743420324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:00,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:00,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:00,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743420324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:00,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:00,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:00,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:00,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:00,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:00,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:00,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743420333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:00,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:00,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:00,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:00,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743420333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:00,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:00,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:00,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:00,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:00,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-27T21:36:00,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:00,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. as already flushing 2024-11-27T21:36:00,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:00,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:00,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:00,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:00,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-27T21:36:00,465 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=92, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/e9f2aed1d7d94d2884265b9ee63e79f5 2024-11-27T21:36:00,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/56cd2647591049e0a6c668cd86401a7b is 50, key is test_row_0/B:col10/1732743359309/Put/seqid=0 2024-11-27T21:36:00,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742198_1374 (size=12001) 2024-11-27T21:36:00,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-27T21:36:00,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:00,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. as already flushing 2024-11-27T21:36:00,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:00,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:00,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:00,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:00,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-27T21:36:00,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:00,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. as already flushing 2024-11-27T21:36:00,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:00,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:00,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:00,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:00,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-27T21:36:00,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:00,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743420833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:00,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:00,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:00,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:00,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-27T21:36:00,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:00,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743420843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:00,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:00,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. as already flushing 2024-11-27T21:36:00,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:00,850 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:00,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:00,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:00,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:00,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:00,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:00,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:00,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743420854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:00,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:00,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:00,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743420854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:00,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:00,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:00,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:00,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:00,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:00,877 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/56cd2647591049e0a6c668cd86401a7b 2024-11-27T21:36:00,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/1b71d38516e844c09a4a6a5cf7aaf22a is 50, key is test_row_0/C:col10/1732743359309/Put/seqid=0 2024-11-27T21:36:00,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742199_1375 (size=12001) 2024-11-27T21:36:00,888 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/1b71d38516e844c09a4a6a5cf7aaf22a 2024-11-27T21:36:00,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/e9f2aed1d7d94d2884265b9ee63e79f5 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/e9f2aed1d7d94d2884265b9ee63e79f5 2024-11-27T21:36:00,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/e9f2aed1d7d94d2884265b9ee63e79f5, entries=250, sequenceid=92, filesize=47.0 K 2024-11-27T21:36:00,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/56cd2647591049e0a6c668cd86401a7b as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/56cd2647591049e0a6c668cd86401a7b 2024-11-27T21:36:00,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/56cd2647591049e0a6c668cd86401a7b, entries=150, sequenceid=92, filesize=11.7 K 2024-11-27T21:36:00,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/1b71d38516e844c09a4a6a5cf7aaf22a as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/1b71d38516e844c09a4a6a5cf7aaf22a 2024-11-27T21:36:00,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/1b71d38516e844c09a4a6a5cf7aaf22a, entries=150, sequenceid=92, filesize=11.7 K 2024-11-27T21:36:00,904 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for f9b9357dca19fe90b610172c1465ef7c in 1268ms, sequenceid=92, compaction requested=true 2024-11-27T21:36:00,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f9b9357dca19fe90b610172c1465ef7c: 2024-11-27T21:36:00,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f9b9357dca19fe90b610172c1465ef7c:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:36:00,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:00,905 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:00,905 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:00,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f9b9357dca19fe90b610172c1465ef7c:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:36:00,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:00,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f9b9357dca19fe90b610172c1465ef7c:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:36:00,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:36:00,905 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110152 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:00,906 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): f9b9357dca19fe90b610172c1465ef7c/A is initiating minor compaction (all files) 2024-11-27T21:36:00,906 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f9b9357dca19fe90b610172c1465ef7c/A in TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:00,906 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/9954eb11b40046679ba847a6d96504ab, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/01c0c77496764d46a126dd00f57db203, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/e9f2aed1d7d94d2884265b9ee63e79f5] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp, totalSize=107.6 K 2024-11-27T21:36:00,906 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:00,906 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. files: [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/9954eb11b40046679ba847a6d96504ab, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/01c0c77496764d46a126dd00f57db203, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/e9f2aed1d7d94d2884265b9ee63e79f5] 2024-11-27T21:36:00,906 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:00,906 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): f9b9357dca19fe90b610172c1465ef7c/B is initiating minor compaction (all files) 2024-11-27T21:36:00,906 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f9b9357dca19fe90b610172c1465ef7c/B in TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:00,906 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/e283383e8a39481ba63bc1ea8b900cb5, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/c7f4756b6f764330bcf0a00cf5ac0a04, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/56cd2647591049e0a6c668cd86401a7b] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp, totalSize=35.3 K 2024-11-27T21:36:00,906 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9954eb11b40046679ba847a6d96504ab, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732743357022 2024-11-27T21:36:00,906 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting e283383e8a39481ba63bc1ea8b900cb5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732743357022 2024-11-27T21:36:00,907 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting c7f4756b6f764330bcf0a00cf5ac0a04, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732743357103 2024-11-27T21:36:00,907 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01c0c77496764d46a126dd00f57db203, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732743357103 2024-11-27T21:36:00,907 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9f2aed1d7d94d2884265b9ee63e79f5, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732743359308 2024-11-27T21:36:00,907 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 56cd2647591049e0a6c668cd86401a7b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732743359308 2024-11-27T21:36:00,914 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f9b9357dca19fe90b610172c1465ef7c#B#compaction#322 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:00,914 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/d7e30b2d41314c36a5fde82a1e805b2a is 50, key is test_row_0/B:col10/1732743359309/Put/seqid=0 2024-11-27T21:36:00,917 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f9b9357dca19fe90b610172c1465ef7c] 2024-11-27T21:36:00,921 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127122e433fa08941c7b0eddbe18f578446_f9b9357dca19fe90b610172c1465ef7c store=[table=TestAcidGuarantees family=A region=f9b9357dca19fe90b610172c1465ef7c] 2024-11-27T21:36:00,922 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127122e433fa08941c7b0eddbe18f578446_f9b9357dca19fe90b610172c1465ef7c, store=[table=TestAcidGuarantees family=A region=f9b9357dca19fe90b610172c1465ef7c] 2024-11-27T21:36:00,922 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127122e433fa08941c7b0eddbe18f578446_f9b9357dca19fe90b610172c1465ef7c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f9b9357dca19fe90b610172c1465ef7c] 2024-11-27T21:36:00,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742200_1376 (size=12207) 2024-11-27T21:36:00,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742201_1377 (size=4469) 2024-11-27T21:36:01,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-27T21:36:01,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:01,003 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing f9b9357dca19fe90b610172c1465ef7c 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-27T21:36:01,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112709982fe646f64c14837fbf754b883958_f9b9357dca19fe90b610172c1465ef7c is 50, key is test_row_0/A:col10/1732743359684/Put/seqid=0 2024-11-27T21:36:01,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742202_1378 (size=12154) 2024-11-27T21:36:01,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:36:01,018 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112709982fe646f64c14837fbf754b883958_f9b9357dca19fe90b610172c1465ef7c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112709982fe646f64c14837fbf754b883958_f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:36:01,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/12aefea8bfd94e098e847ddc04a0224c, store: [table=TestAcidGuarantees family=A region=f9b9357dca19fe90b610172c1465ef7c] 2024-11-27T21:36:01,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/12aefea8bfd94e098e847ddc04a0224c is 175, key is test_row_0/A:col10/1732743359684/Put/seqid=0 2024-11-27T21:36:01,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742203_1379 (size=30955) 2024-11-27T21:36:01,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-27T21:36:01,328 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/d7e30b2d41314c36a5fde82a1e805b2a as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/d7e30b2d41314c36a5fde82a1e805b2a 2024-11-27T21:36:01,330 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f9b9357dca19fe90b610172c1465ef7c#A#compaction#323 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:01,331 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/fab7050e16d24d95b6ca0f41190b92a8 is 175, key is test_row_0/A:col10/1732743359309/Put/seqid=0 2024-11-27T21:36:01,333 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f9b9357dca19fe90b610172c1465ef7c/B of f9b9357dca19fe90b610172c1465ef7c into d7e30b2d41314c36a5fde82a1e805b2a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:01,333 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f9b9357dca19fe90b610172c1465ef7c: 2024-11-27T21:36:01,333 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., storeName=f9b9357dca19fe90b610172c1465ef7c/B, priority=13, startTime=1732743360905; duration=0sec 2024-11-27T21:36:01,334 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:36:01,334 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f9b9357dca19fe90b610172c1465ef7c:B 2024-11-27T21:36:01,334 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:01,335 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:01,335 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): f9b9357dca19fe90b610172c1465ef7c/C is initiating minor compaction (all files) 2024-11-27T21:36:01,335 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f9b9357dca19fe90b610172c1465ef7c/C in TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:01,335 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/bc050f7cbf2e4c83867bd8acad07e40d, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/0d14d18731b0430facbaf7e0d6525c8e, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/1b71d38516e844c09a4a6a5cf7aaf22a] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp, totalSize=35.3 K 2024-11-27T21:36:01,335 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting bc050f7cbf2e4c83867bd8acad07e40d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732743357022 2024-11-27T21:36:01,336 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d14d18731b0430facbaf7e0d6525c8e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732743357103 2024-11-27T21:36:01,336 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b71d38516e844c09a4a6a5cf7aaf22a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732743359308 2024-11-27T21:36:01,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:36:01,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. as already flushing 2024-11-27T21:36:01,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742204_1380 (size=31161) 2024-11-27T21:36:01,355 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/fab7050e16d24d95b6ca0f41190b92a8 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/fab7050e16d24d95b6ca0f41190b92a8 2024-11-27T21:36:01,355 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f9b9357dca19fe90b610172c1465ef7c#C#compaction#325 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:01,355 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/2ee152e6a5a94e789ab6871afbb13542 is 50, key is test_row_0/C:col10/1732743359309/Put/seqid=0 2024-11-27T21:36:01,359 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f9b9357dca19fe90b610172c1465ef7c/A of f9b9357dca19fe90b610172c1465ef7c into fab7050e16d24d95b6ca0f41190b92a8(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:01,359 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f9b9357dca19fe90b610172c1465ef7c: 2024-11-27T21:36:01,359 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., storeName=f9b9357dca19fe90b610172c1465ef7c/A, priority=13, startTime=1732743360905; duration=0sec 2024-11-27T21:36:01,359 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:01,359 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f9b9357dca19fe90b610172c1465ef7c:A 2024-11-27T21:36:01,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742205_1381 (size=12207) 2024-11-27T21:36:01,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:01,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743421390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:01,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:01,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:01,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:01,425 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=115, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/12aefea8bfd94e098e847ddc04a0224c 2024-11-27T21:36:01,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/f8891a4503864266aea1ab287e48f495 is 50, key is test_row_0/B:col10/1732743359684/Put/seqid=0 2024-11-27T21:36:01,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742206_1382 (size=12001) 2024-11-27T21:36:01,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:01,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743421503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:01,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:01,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:01,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:01,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:01,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743421724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:01,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:01,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:01,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:01,770 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/2ee152e6a5a94e789ab6871afbb13542 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/2ee152e6a5a94e789ab6871afbb13542 2024-11-27T21:36:01,775 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f9b9357dca19fe90b610172c1465ef7c/C of f9b9357dca19fe90b610172c1465ef7c into 2ee152e6a5a94e789ab6871afbb13542(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:01,775 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f9b9357dca19fe90b610172c1465ef7c: 2024-11-27T21:36:01,775 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., storeName=f9b9357dca19fe90b610172c1465ef7c/C, priority=13, startTime=1732743360905; duration=0sec 2024-11-27T21:36:01,775 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:01,775 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f9b9357dca19fe90b610172c1465ef7c:C 2024-11-27T21:36:01,838 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/f8891a4503864266aea1ab287e48f495 2024-11-27T21:36:01,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/788289743849467fb62fe7c2f9f8d197 is 50, key is test_row_0/C:col10/1732743359684/Put/seqid=0 2024-11-27T21:36:01,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:01,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743421844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:01,848 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,39097,1732743267048 for 'test_row_0' in f9b9357dca19fe90b610172c1465ef7c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:36:01,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:01,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:01,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:01,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742207_1383 (size=12001) 2024-11-27T21:36:01,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:01,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743421863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:01,869 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,39097,1732743267048 for 'test_row_0' in f9b9357dca19fe90b610172c1465ef7c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:36:01,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:01,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:01,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:01,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:01,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743421873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:01,880 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,39097,1732743267048 for 'test_row_2' in f9b9357dca19fe90b610172c1465ef7c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:36:01,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:01,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:01,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:01,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:01,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743421883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:01,891 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,39097,1732743267048 for 'test_row_1' in f9b9357dca19fe90b610172c1465ef7c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:36:01,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:01,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:01,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:02,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:02,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743422044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:02,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:02,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:02,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:02,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-27T21:36:02,251 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/788289743849467fb62fe7c2f9f8d197 2024-11-27T21:36:02,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/12aefea8bfd94e098e847ddc04a0224c as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/12aefea8bfd94e098e847ddc04a0224c 2024-11-27T21:36:02,259 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/12aefea8bfd94e098e847ddc04a0224c, entries=150, sequenceid=115, filesize=30.2 K 2024-11-27T21:36:02,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/f8891a4503864266aea1ab287e48f495 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/f8891a4503864266aea1ab287e48f495 2024-11-27T21:36:02,263 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/f8891a4503864266aea1ab287e48f495, entries=150, sequenceid=115, filesize=11.7 K 2024-11-27T21:36:02,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/788289743849467fb62fe7c2f9f8d197 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/788289743849467fb62fe7c2f9f8d197 2024-11-27T21:36:02,267 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/788289743849467fb62fe7c2f9f8d197, entries=150, sequenceid=115, filesize=11.7 K 2024-11-27T21:36:02,268 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for f9b9357dca19fe90b610172c1465ef7c in 1265ms, sequenceid=115, compaction requested=false 2024-11-27T21:36:02,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for f9b9357dca19fe90b610172c1465ef7c: 2024-11-27T21:36:02,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:02,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-27T21:36:02,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-27T21:36:02,271 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-27T21:36:02,271 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1840 sec 2024-11-27T21:36:02,273 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 2.1890 sec 2024-11-27T21:36:02,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:36:02,559 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:36:02,559 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f9b9357dca19fe90b610172c1465ef7c 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-27T21:36:02,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112748640639c91b4fa68a740821c0aaed0f_f9b9357dca19fe90b610172c1465ef7c is 50, key is test_row_0/A:col10/1732743361380/Put/seqid=0 2024-11-27T21:36:02,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742208_1384 (size=14694) 2024-11-27T21:36:02,572 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:36:02,575 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112748640639c91b4fa68a740821c0aaed0f_f9b9357dca19fe90b610172c1465ef7c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112748640639c91b4fa68a740821c0aaed0f_f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:36:02,576 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/d6967770ad8d4a90af4cc0d47c8dc2fc, store: [table=TestAcidGuarantees family=A region=f9b9357dca19fe90b610172c1465ef7c] 2024-11-27T21:36:02,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/d6967770ad8d4a90af4cc0d47c8dc2fc is 175, key is test_row_0/A:col10/1732743361380/Put/seqid=0 2024-11-27T21:36:02,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742209_1385 (size=39649) 2024-11-27T21:36:02,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:02,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743422737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:02,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:02,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:02,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:02,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:02,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743422853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:02,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:02,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:02,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:02,986 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/d6967770ad8d4a90af4cc0d47c8dc2fc 2024-11-27T21:36:02,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/3b998edb0ea046ee8f95a3619b54d312 is 50, key is test_row_0/B:col10/1732743361380/Put/seqid=0 2024-11-27T21:36:02,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742210_1386 (size=12051) 2024-11-27T21:36:03,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:03,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743423063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:03,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:03,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:03,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:03,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:03,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743423373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:03,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:03,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:03,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:03,397 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/3b998edb0ea046ee8f95a3619b54d312 2024-11-27T21:36:03,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/a411f23c3a0c4c9694eb1f70c95d822c is 50, key is test_row_0/C:col10/1732743361380/Put/seqid=0 2024-11-27T21:36:03,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742211_1387 (size=12051) 2024-11-27T21:36:03,810 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/a411f23c3a0c4c9694eb1f70c95d822c 2024-11-27T21:36:03,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/d6967770ad8d4a90af4cc0d47c8dc2fc as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/d6967770ad8d4a90af4cc0d47c8dc2fc 2024-11-27T21:36:03,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/d6967770ad8d4a90af4cc0d47c8dc2fc, entries=200, sequenceid=132, filesize=38.7 K 2024-11-27T21:36:03,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/3b998edb0ea046ee8f95a3619b54d312 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/3b998edb0ea046ee8f95a3619b54d312 2024-11-27T21:36:03,826 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/3b998edb0ea046ee8f95a3619b54d312, entries=150, sequenceid=132, filesize=11.8 K 2024-11-27T21:36:03,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/a411f23c3a0c4c9694eb1f70c95d822c as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/a411f23c3a0c4c9694eb1f70c95d822c 2024-11-27T21:36:03,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/a411f23c3a0c4c9694eb1f70c95d822c, entries=150, sequenceid=132, filesize=11.8 K 2024-11-27T21:36:03,833 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for f9b9357dca19fe90b610172c1465ef7c in 1274ms, sequenceid=132, compaction requested=true 2024-11-27T21:36:03,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f9b9357dca19fe90b610172c1465ef7c: 2024-11-27T21:36:03,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f9b9357dca19fe90b610172c1465ef7c:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:36:03,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:03,833 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:03,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f9b9357dca19fe90b610172c1465ef7c:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:36:03,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:03,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f9b9357dca19fe90b610172c1465ef7c:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:36:03,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:36:03,833 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:03,834 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:03,834 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): f9b9357dca19fe90b610172c1465ef7c/B is initiating minor compaction (all files) 2024-11-27T21:36:03,834 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101765 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:03,834 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): f9b9357dca19fe90b610172c1465ef7c/A is initiating minor compaction (all files) 2024-11-27T21:36:03,834 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f9b9357dca19fe90b610172c1465ef7c/B in TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:03,834 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f9b9357dca19fe90b610172c1465ef7c/A in TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:03,834 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/d7e30b2d41314c36a5fde82a1e805b2a, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/f8891a4503864266aea1ab287e48f495, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/3b998edb0ea046ee8f95a3619b54d312] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp, totalSize=35.4 K 2024-11-27T21:36:03,834 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/fab7050e16d24d95b6ca0f41190b92a8, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/12aefea8bfd94e098e847ddc04a0224c, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/d6967770ad8d4a90af4cc0d47c8dc2fc] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp, totalSize=99.4 K 2024-11-27T21:36:03,834 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:03,834 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. files: [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/fab7050e16d24d95b6ca0f41190b92a8, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/12aefea8bfd94e098e847ddc04a0224c, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/d6967770ad8d4a90af4cc0d47c8dc2fc] 2024-11-27T21:36:03,835 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting d7e30b2d41314c36a5fde82a1e805b2a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732743359308 2024-11-27T21:36:03,835 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting fab7050e16d24d95b6ca0f41190b92a8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732743359308 2024-11-27T21:36:03,835 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting f8891a4503864266aea1ab287e48f495, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732743359684 2024-11-27T21:36:03,835 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12aefea8bfd94e098e847ddc04a0224c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732743359684 2024-11-27T21:36:03,835 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b998edb0ea046ee8f95a3619b54d312, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732743361358 2024-11-27T21:36:03,835 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6967770ad8d4a90af4cc0d47c8dc2fc, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732743361358 2024-11-27T21:36:03,851 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f9b9357dca19fe90b610172c1465ef7c#B#compaction#331 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:03,852 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/8d3d8c10b8db4c65863f364c02bfebac is 50, key is test_row_0/B:col10/1732743361380/Put/seqid=0 2024-11-27T21:36:03,853 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f9b9357dca19fe90b610172c1465ef7c] 2024-11-27T21:36:03,858 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411271a80932f844c46b2b7565fdafda050e0_f9b9357dca19fe90b610172c1465ef7c store=[table=TestAcidGuarantees family=A region=f9b9357dca19fe90b610172c1465ef7c] 2024-11-27T21:36:03,859 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411271a80932f844c46b2b7565fdafda050e0_f9b9357dca19fe90b610172c1465ef7c, store=[table=TestAcidGuarantees family=A region=f9b9357dca19fe90b610172c1465ef7c] 2024-11-27T21:36:03,860 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411271a80932f844c46b2b7565fdafda050e0_f9b9357dca19fe90b610172c1465ef7c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f9b9357dca19fe90b610172c1465ef7c] 2024-11-27T21:36:03,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742212_1388 (size=12359) 2024-11-27T21:36:03,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:36:03,867 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:36:03,868 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f9b9357dca19fe90b610172c1465ef7c 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-27T21:36:03,873 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/8d3d8c10b8db4c65863f364c02bfebac as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/8d3d8c10b8db4c65863f364c02bfebac 2024-11-27T21:36:03,877 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f9b9357dca19fe90b610172c1465ef7c/B of f9b9357dca19fe90b610172c1465ef7c into 8d3d8c10b8db4c65863f364c02bfebac(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:03,877 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f9b9357dca19fe90b610172c1465ef7c: 2024-11-27T21:36:03,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411275d7564d5169e44589334a0e5338c3ca9_f9b9357dca19fe90b610172c1465ef7c is 50, key is test_row_0/A:col10/1732743363864/Put/seqid=0 2024-11-27T21:36:03,877 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., storeName=f9b9357dca19fe90b610172c1465ef7c/B, priority=13, startTime=1732743363833; duration=0sec 2024-11-27T21:36:03,877 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:36:03,877 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f9b9357dca19fe90b610172c1465ef7c:B 2024-11-27T21:36:03,877 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:03,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742213_1389 (size=4469) 2024-11-27T21:36:03,879 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:03,879 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): f9b9357dca19fe90b610172c1465ef7c/C is initiating minor compaction (all files) 2024-11-27T21:36:03,879 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f9b9357dca19fe90b610172c1465ef7c/C in TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:03,879 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/2ee152e6a5a94e789ab6871afbb13542, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/788289743849467fb62fe7c2f9f8d197, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/a411f23c3a0c4c9694eb1f70c95d822c] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp, totalSize=35.4 K 2024-11-27T21:36:03,879 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ee152e6a5a94e789ab6871afbb13542, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732743359308 2024-11-27T21:36:03,879 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 788289743849467fb62fe7c2f9f8d197, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732743359684 2024-11-27T21:36:03,880 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting a411f23c3a0c4c9694eb1f70c95d822c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732743361358 2024-11-27T21:36:03,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742214_1390 (size=14794) 2024-11-27T21:36:03,891 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f9b9357dca19fe90b610172c1465ef7c#C#compaction#334 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:03,892 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/90bcbc96ce3f4c2ba035e6d3f22f4f02 is 50, key is test_row_0/C:col10/1732743361380/Put/seqid=0 2024-11-27T21:36:03,893 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:36:03,900 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411275d7564d5169e44589334a0e5338c3ca9_f9b9357dca19fe90b610172c1465ef7c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411275d7564d5169e44589334a0e5338c3ca9_f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:36:03,901 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/76a32d2d0e044604a719cf87a4d564af, store: [table=TestAcidGuarantees family=A region=f9b9357dca19fe90b610172c1465ef7c] 2024-11-27T21:36:03,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/76a32d2d0e044604a719cf87a4d564af is 175, key is test_row_0/A:col10/1732743363864/Put/seqid=0 2024-11-27T21:36:03,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742215_1391 (size=12359) 2024-11-27T21:36:03,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742216_1392 (size=39749) 2024-11-27T21:36:03,921 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/76a32d2d0e044604a719cf87a4d564af 2024-11-27T21:36:03,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:03,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743423919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:03,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:03,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:03,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:03,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/c8fcfe69faee4666a3320a31410cbf62 is 50, key is test_row_0/B:col10/1732743363864/Put/seqid=0 2024-11-27T21:36:03,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:03,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743423922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:03,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:03,937 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:03,937 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:03,937 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:03,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743423922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:03,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:03,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:03,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:03,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:03,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743423922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:03,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:03,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:03,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:03,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:03,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743423924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:03,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:03,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:03,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:03,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742217_1393 (size=12151) 2024-11-27T21:36:03,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/c8fcfe69faee4666a3320a31410cbf62 2024-11-27T21:36:03,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/cd61ac3a34524d13b3140a41d589a8e1 is 50, key is test_row_0/C:col10/1732743363864/Put/seqid=0 2024-11-27T21:36:03,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742218_1394 (size=12151) 2024-11-27T21:36:03,964 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/cd61ac3a34524d13b3140a41d589a8e1 2024-11-27T21:36:03,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/76a32d2d0e044604a719cf87a4d564af as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/76a32d2d0e044604a719cf87a4d564af 2024-11-27T21:36:03,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/76a32d2d0e044604a719cf87a4d564af, entries=200, sequenceid=155, filesize=38.8 K 2024-11-27T21:36:03,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/c8fcfe69faee4666a3320a31410cbf62 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/c8fcfe69faee4666a3320a31410cbf62 2024-11-27T21:36:03,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/c8fcfe69faee4666a3320a31410cbf62, entries=150, sequenceid=155, filesize=11.9 K 2024-11-27T21:36:03,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/cd61ac3a34524d13b3140a41d589a8e1 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/cd61ac3a34524d13b3140a41d589a8e1 2024-11-27T21:36:03,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/cd61ac3a34524d13b3140a41d589a8e1, entries=150, sequenceid=155, filesize=11.9 K 2024-11-27T21:36:03,986 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for f9b9357dca19fe90b610172c1465ef7c in 119ms, sequenceid=155, compaction requested=false 2024-11-27T21:36:03,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f9b9357dca19fe90b610172c1465ef7c: 2024-11-27T21:36:04,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:36:04,039 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:36:04,040 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f9b9357dca19fe90b610172c1465ef7c 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-27T21:36:04,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127fe336c09412042af9a326821d1379087_f9b9357dca19fe90b610172c1465ef7c is 50, key is test_row_0/A:col10/1732743363921/Put/seqid=0 2024-11-27T21:36:04,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742219_1395 (size=14794) 2024-11-27T21:36:04,060 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:36:04,064 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127fe336c09412042af9a326821d1379087_f9b9357dca19fe90b610172c1465ef7c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127fe336c09412042af9a326821d1379087_f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:36:04,065 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/be57d82bdc294942aba81affb5b84302, store: [table=TestAcidGuarantees family=A region=f9b9357dca19fe90b610172c1465ef7c] 2024-11-27T21:36:04,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/be57d82bdc294942aba81affb5b84302 is 175, key is test_row_0/A:col10/1732743363921/Put/seqid=0 2024-11-27T21:36:04,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742220_1396 (size=39749) 2024-11-27T21:36:04,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:04,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743424085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:04,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:04,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:04,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:04,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:04,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743424086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:04,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:04,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:04,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:04,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:04,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743424087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:04,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:04,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743424088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:04,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:04,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:04,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:04,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:04,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743424089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:04,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:04,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:04,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:04,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:04,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:04,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:04,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:04,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743424193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:04,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:04,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:04,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:04,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:04,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743424203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:04,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:04,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:04,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:04,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:04,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743424203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:04,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:04,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:04,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:04,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:04,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743424203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:04,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:04,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:04,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:04,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:04,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743424213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:04,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:04,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:04,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:04,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-27T21:36:04,253 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:36:04,254 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:36:04,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-27T21:36:04,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-27T21:36:04,255 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:36:04,256 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:36:04,256 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:36:04,279 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f9b9357dca19fe90b610172c1465ef7c#A#compaction#332 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:04,280 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/4a40798050c94b1ba7b98c12f95a2f5d is 175, key is test_row_0/A:col10/1732743361380/Put/seqid=0 2024-11-27T21:36:04,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742221_1397 (size=31313) 2024-11-27T21:36:04,318 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/90bcbc96ce3f4c2ba035e6d3f22f4f02 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/90bcbc96ce3f4c2ba035e6d3f22f4f02 2024-11-27T21:36:04,322 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f9b9357dca19fe90b610172c1465ef7c/C of f9b9357dca19fe90b610172c1465ef7c into 90bcbc96ce3f4c2ba035e6d3f22f4f02(size=12.1 K), total size for store is 23.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:04,322 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f9b9357dca19fe90b610172c1465ef7c: 2024-11-27T21:36:04,322 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., storeName=f9b9357dca19fe90b610172c1465ef7c/C, priority=13, startTime=1732743363833; duration=0sec 2024-11-27T21:36:04,322 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:04,323 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f9b9357dca19fe90b610172c1465ef7c:C 2024-11-27T21:36:04,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-27T21:36:04,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-27T21:36:04,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:04,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. as already flushing 2024-11-27T21:36:04,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:04,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:04,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:04,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:04,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:04,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743424403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:04,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:04,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:04,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:04,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:04,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743424424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:04,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:04,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:04,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743424424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:04,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:04,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:04,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:04,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:04,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:04,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:04,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743424424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:04,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:04,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:04,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:04,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:04,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743424433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:04,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:04,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:04,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:04,469 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/be57d82bdc294942aba81affb5b84302 2024-11-27T21:36:04,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/3f83c44b6e4d4af190382807ee63d5e9 is 50, key is test_row_0/B:col10/1732743363921/Put/seqid=0 2024-11-27T21:36:04,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742222_1398 (size=12151) 2024-11-27T21:36:04,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-27T21:36:04,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:04,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. as already flushing 2024-11-27T21:36:04,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:04,562 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:04,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:04,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:04,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-27T21:36:04,688 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/4a40798050c94b1ba7b98c12f95a2f5d as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/4a40798050c94b1ba7b98c12f95a2f5d 2024-11-27T21:36:04,692 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f9b9357dca19fe90b610172c1465ef7c/A of f9b9357dca19fe90b610172c1465ef7c into 4a40798050c94b1ba7b98c12f95a2f5d(size=30.6 K), total size for store is 69.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:04,692 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f9b9357dca19fe90b610172c1465ef7c: 2024-11-27T21:36:04,692 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., storeName=f9b9357dca19fe90b610172c1465ef7c/A, priority=13, startTime=1732743363833; duration=0sec 2024-11-27T21:36:04,692 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:04,692 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f9b9357dca19fe90b610172c1465ef7c:A 2024-11-27T21:36:04,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-27T21:36:04,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:04,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. as already flushing 2024-11-27T21:36:04,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:04,715 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:04,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:04,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:04,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:04,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743424723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:04,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:04,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:04,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:04,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:04,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743424733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:04,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:04,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:04,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:04,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:04,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743424744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:04,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:04,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743424744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:04,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:04,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:04,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:04,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:04,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:04,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:04,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:04,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743424753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:04,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:04,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:04,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:04,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-27T21:36:04,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:04,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. as already flushing 2024-11-27T21:36:04,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:04,867 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:04,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:04,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:04,880 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/3f83c44b6e4d4af190382807ee63d5e9 2024-11-27T21:36:04,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-27T21:36:04,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/e614ff4d32ba44bca26aed5ccd574c74 is 50, key is test_row_0/C:col10/1732743363921/Put/seqid=0 2024-11-27T21:36:04,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742223_1399 (size=12151) 2024-11-27T21:36:04,892 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/e614ff4d32ba44bca26aed5ccd574c74 2024-11-27T21:36:04,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/be57d82bdc294942aba81affb5b84302 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/be57d82bdc294942aba81affb5b84302 2024-11-27T21:36:04,899 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/be57d82bdc294942aba81affb5b84302, entries=200, sequenceid=170, filesize=38.8 K 2024-11-27T21:36:04,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/3f83c44b6e4d4af190382807ee63d5e9 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/3f83c44b6e4d4af190382807ee63d5e9 2024-11-27T21:36:04,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/3f83c44b6e4d4af190382807ee63d5e9, entries=150, sequenceid=170, filesize=11.9 K 2024-11-27T21:36:04,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/e614ff4d32ba44bca26aed5ccd574c74 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/e614ff4d32ba44bca26aed5ccd574c74 2024-11-27T21:36:04,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/e614ff4d32ba44bca26aed5ccd574c74, entries=150, sequenceid=170, filesize=11.9 K 2024-11-27T21:36:04,908 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for f9b9357dca19fe90b610172c1465ef7c in 869ms, sequenceid=170, compaction requested=true 2024-11-27T21:36:04,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f9b9357dca19fe90b610172c1465ef7c: 2024-11-27T21:36:04,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f9b9357dca19fe90b610172c1465ef7c:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:36:04,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:04,908 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:04,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f9b9357dca19fe90b610172c1465ef7c:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:36:04,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:04,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f9b9357dca19fe90b610172c1465ef7c:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:36:04,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:36:04,908 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:04,909 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110811 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:04,909 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:04,909 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): f9b9357dca19fe90b610172c1465ef7c/A is initiating minor compaction (all files) 2024-11-27T21:36:04,909 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): f9b9357dca19fe90b610172c1465ef7c/B is initiating minor compaction (all files) 2024-11-27T21:36:04,909 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f9b9357dca19fe90b610172c1465ef7c/A in TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:04,909 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f9b9357dca19fe90b610172c1465ef7c/B in TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:04,909 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/8d3d8c10b8db4c65863f364c02bfebac, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/c8fcfe69faee4666a3320a31410cbf62, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/3f83c44b6e4d4af190382807ee63d5e9] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp, totalSize=35.8 K 2024-11-27T21:36:04,909 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/4a40798050c94b1ba7b98c12f95a2f5d, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/76a32d2d0e044604a719cf87a4d564af, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/be57d82bdc294942aba81affb5b84302] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp, totalSize=108.2 K 2024-11-27T21:36:04,909 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:04,909 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. files: [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/4a40798050c94b1ba7b98c12f95a2f5d, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/76a32d2d0e044604a719cf87a4d564af, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/be57d82bdc294942aba81affb5b84302] 2024-11-27T21:36:04,910 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d3d8c10b8db4c65863f364c02bfebac, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732743361358 2024-11-27T21:36:04,910 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a40798050c94b1ba7b98c12f95a2f5d, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732743361358 2024-11-27T21:36:04,910 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting c8fcfe69faee4666a3320a31410cbf62, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732743362717 2024-11-27T21:36:04,910 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76a32d2d0e044604a719cf87a4d564af, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732743362717 2024-11-27T21:36:04,910 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f83c44b6e4d4af190382807ee63d5e9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732743363917 2024-11-27T21:36:04,910 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting be57d82bdc294942aba81affb5b84302, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732743363917 2024-11-27T21:36:04,918 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f9b9357dca19fe90b610172c1465ef7c#B#compaction#340 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:04,919 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/13bd8897c45741d79707d3e4bc4eb1d9 is 50, key is test_row_0/B:col10/1732743363921/Put/seqid=0 2024-11-27T21:36:04,920 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f9b9357dca19fe90b610172c1465ef7c] 2024-11-27T21:36:04,931 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127b06781760b5445f0a68362f9fe9fbed4_f9b9357dca19fe90b610172c1465ef7c store=[table=TestAcidGuarantees family=A region=f9b9357dca19fe90b610172c1465ef7c] 2024-11-27T21:36:04,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742224_1400 (size=12561) 2024-11-27T21:36:04,933 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127b06781760b5445f0a68362f9fe9fbed4_f9b9357dca19fe90b610172c1465ef7c, store=[table=TestAcidGuarantees family=A region=f9b9357dca19fe90b610172c1465ef7c] 2024-11-27T21:36:04,933 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127b06781760b5445f0a68362f9fe9fbed4_f9b9357dca19fe90b610172c1465ef7c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f9b9357dca19fe90b610172c1465ef7c] 2024-11-27T21:36:04,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742225_1401 (size=4469) 2024-11-27T21:36:05,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-27T21:36:05,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:05,020 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing f9b9357dca19fe90b610172c1465ef7c 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-27T21:36:05,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d3edbdeea7b545f384116330929fd158_f9b9357dca19fe90b610172c1465ef7c is 50, key is test_row_0/A:col10/1732743364088/Put/seqid=0 2024-11-27T21:36:05,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742226_1402 (size=12304) 2024-11-27T21:36:05,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:36:05,035 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d3edbdeea7b545f384116330929fd158_f9b9357dca19fe90b610172c1465ef7c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d3edbdeea7b545f384116330929fd158_f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:36:05,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/ea919ac1eaa840ad9881fa5405a5b018, store: [table=TestAcidGuarantees family=A region=f9b9357dca19fe90b610172c1465ef7c] 2024-11-27T21:36:05,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/ea919ac1eaa840ad9881fa5405a5b018 is 175, key is test_row_0/A:col10/1732743364088/Put/seqid=0 2024-11-27T21:36:05,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742227_1403 (size=31105) 2024-11-27T21:36:05,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:36:05,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. as already flushing 2024-11-27T21:36:05,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:05,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743425277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:05,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:05,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:05,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:05,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:05,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743425278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:05,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:05,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:05,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743425280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:05,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:05,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:05,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:05,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:05,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:05,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:05,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743425282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:05,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:05,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:05,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:05,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743425283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:05,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:05,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:05,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:05,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:05,336 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/13bd8897c45741d79707d3e4bc4eb1d9 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/13bd8897c45741d79707d3e4bc4eb1d9 2024-11-27T21:36:05,340 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f9b9357dca19fe90b610172c1465ef7c/B of f9b9357dca19fe90b610172c1465ef7c into 13bd8897c45741d79707d3e4bc4eb1d9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:05,340 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f9b9357dca19fe90b610172c1465ef7c: 2024-11-27T21:36:05,341 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., storeName=f9b9357dca19fe90b610172c1465ef7c/B, priority=13, startTime=1732743364908; duration=0sec 2024-11-27T21:36:05,341 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:36:05,341 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f9b9357dca19fe90b610172c1465ef7c:B 2024-11-27T21:36:05,341 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:05,341 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:05,341 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): f9b9357dca19fe90b610172c1465ef7c/C is initiating minor compaction (all files) 2024-11-27T21:36:05,341 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f9b9357dca19fe90b610172c1465ef7c/C in TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:05,342 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/90bcbc96ce3f4c2ba035e6d3f22f4f02, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/cd61ac3a34524d13b3140a41d589a8e1, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/e614ff4d32ba44bca26aed5ccd574c74] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp, totalSize=35.8 K 2024-11-27T21:36:05,342 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 90bcbc96ce3f4c2ba035e6d3f22f4f02, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732743361358 2024-11-27T21:36:05,342 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting cd61ac3a34524d13b3140a41d589a8e1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732743362717 2024-11-27T21:36:05,343 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting e614ff4d32ba44bca26aed5ccd574c74, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732743363917 2024-11-27T21:36:05,345 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f9b9357dca19fe90b610172c1465ef7c#A#compaction#341 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:05,346 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/1dbe25d1aeef479d9ff1f119d3486c11 is 175, key is test_row_0/A:col10/1732743363921/Put/seqid=0 2024-11-27T21:36:05,352 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f9b9357dca19fe90b610172c1465ef7c#C#compaction#343 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:05,353 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/1f19438ae4d54c7091ad1a0c1b910892 is 50, key is test_row_0/C:col10/1732743363921/Put/seqid=0 2024-11-27T21:36:05,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742228_1404 (size=31515) 2024-11-27T21:36:05,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742229_1405 (size=12561) 2024-11-27T21:36:05,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-27T21:36:05,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:05,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743425394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:05,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:05,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:05,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743425394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:05,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:05,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:05,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:05,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:05,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:05,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743425394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:05,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:05,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:05,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:05,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:05,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:05,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743425403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:05,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:05,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743425403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:05,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:05,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:05,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:05,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:05,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:05,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:05,441 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/ea919ac1eaa840ad9881fa5405a5b018 2024-11-27T21:36:05,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/9844b9e7749c4b4e896da6a08a09163f is 50, key is test_row_0/B:col10/1732743364088/Put/seqid=0 2024-11-27T21:36:05,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742230_1406 (size=12151) 2024-11-27T21:36:05,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:05,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743425613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:05,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:05,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:05,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:05,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:05,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743425613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:05,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:05,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:05,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:05,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743425613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:05,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:05,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:05,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:05,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:05,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:05,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743425613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:05,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:05,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:05,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:05,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:05,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743425613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:05,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:05,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:05,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:05,761 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/1dbe25d1aeef479d9ff1f119d3486c11 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/1dbe25d1aeef479d9ff1f119d3486c11 2024-11-27T21:36:05,763 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/1f19438ae4d54c7091ad1a0c1b910892 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/1f19438ae4d54c7091ad1a0c1b910892 2024-11-27T21:36:05,766 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f9b9357dca19fe90b610172c1465ef7c/A of f9b9357dca19fe90b610172c1465ef7c into 1dbe25d1aeef479d9ff1f119d3486c11(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:05,766 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f9b9357dca19fe90b610172c1465ef7c: 2024-11-27T21:36:05,766 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., storeName=f9b9357dca19fe90b610172c1465ef7c/A, priority=13, startTime=1732743364908; duration=0sec 2024-11-27T21:36:05,766 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:05,766 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f9b9357dca19fe90b610172c1465ef7c:A 2024-11-27T21:36:05,767 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f9b9357dca19fe90b610172c1465ef7c/C of f9b9357dca19fe90b610172c1465ef7c into 1f19438ae4d54c7091ad1a0c1b910892(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:05,767 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f9b9357dca19fe90b610172c1465ef7c: 2024-11-27T21:36:05,767 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., storeName=f9b9357dca19fe90b610172c1465ef7c/C, priority=13, startTime=1732743364908; duration=0sec 2024-11-27T21:36:05,767 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:05,767 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f9b9357dca19fe90b610172c1465ef7c:C 2024-11-27T21:36:05,852 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/9844b9e7749c4b4e896da6a08a09163f 2024-11-27T21:36:05,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/9f61529dfd16420a8f5b2a7bc7818b48 is 50, key is test_row_0/C:col10/1732743364088/Put/seqid=0 2024-11-27T21:36:05,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742231_1407 (size=12151) 2024-11-27T21:36:05,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:05,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743425933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:05,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:05,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:05,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:05,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:05,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743425933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:05,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:05,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:05,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:05,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:05,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743425933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:05,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:05,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:05,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:05,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:05,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743425933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:05,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:05,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:05,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:05,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743425933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:05,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:05,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:05,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:05,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:06,264 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/9f61529dfd16420a8f5b2a7bc7818b48 2024-11-27T21:36:06,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/ea919ac1eaa840ad9881fa5405a5b018 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/ea919ac1eaa840ad9881fa5405a5b018 2024-11-27T21:36:06,272 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/ea919ac1eaa840ad9881fa5405a5b018, entries=150, sequenceid=194, filesize=30.4 K 2024-11-27T21:36:06,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/9844b9e7749c4b4e896da6a08a09163f as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/9844b9e7749c4b4e896da6a08a09163f 2024-11-27T21:36:06,276 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/9844b9e7749c4b4e896da6a08a09163f, entries=150, sequenceid=194, filesize=11.9 K 2024-11-27T21:36:06,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/9f61529dfd16420a8f5b2a7bc7818b48 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/9f61529dfd16420a8f5b2a7bc7818b48 2024-11-27T21:36:06,279 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/9f61529dfd16420a8f5b2a7bc7818b48, entries=150, sequenceid=194, filesize=11.9 K 2024-11-27T21:36:06,280 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for f9b9357dca19fe90b610172c1465ef7c in 1260ms, sequenceid=194, compaction requested=false 2024-11-27T21:36:06,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for f9b9357dca19fe90b610172c1465ef7c: 2024-11-27T21:36:06,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:06,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-27T21:36:06,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-27T21:36:06,283 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-27T21:36:06,283 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0250 sec 2024-11-27T21:36:06,284 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 2.0290 sec 2024-11-27T21:36:06,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-27T21:36:06,404 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:36:06,404 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:36:06,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-27T21:36:06,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-27T21:36:06,406 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:36:06,407 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:36:06,407 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:36:06,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:36:06,450 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:36:06,450 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f9b9357dca19fe90b610172c1465ef7c 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-27T21:36:06,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127e43ad6d641184304b3abc2adb76da562_f9b9357dca19fe90b610172c1465ef7c is 50, key is test_row_0/A:col10/1732743366450/Put/seqid=0 2024-11-27T21:36:06,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742232_1408 (size=17284) 2024-11-27T21:36:06,462 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:36:06,466 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127e43ad6d641184304b3abc2adb76da562_f9b9357dca19fe90b610172c1465ef7c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127e43ad6d641184304b3abc2adb76da562_f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:36:06,466 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/d33616f433904988a4895f47b43ad6b9, store: [table=TestAcidGuarantees family=A region=f9b9357dca19fe90b610172c1465ef7c] 2024-11-27T21:36:06,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/d33616f433904988a4895f47b43ad6b9 is 175, key is test_row_0/A:col10/1732743366450/Put/seqid=0 2024-11-27T21:36:06,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742233_1409 (size=48389) 2024-11-27T21:36:06,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-27T21:36:06,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:06,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743426515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:06,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:06,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743426517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:06,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:06,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:06,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:06,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:06,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:06,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743426517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:06,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:06,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:06,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:06,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:06,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:06,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:06,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743426530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:06,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:06,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:06,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:06,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:06,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743426531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:06,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:06,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:06,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:06,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-27T21:36:06,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:06,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. as already flushing 2024-11-27T21:36:06,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:06,559 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:06,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:06,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:06,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:06,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743426644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:06,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:06,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:06,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743426644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:06,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:06,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:06,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:06,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743426644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:06,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:06,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:06,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:06,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:06,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743426644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:06,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:06,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:06,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:06,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:06,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:06,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:06,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:06,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743426653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:06,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:06,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:06,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:06,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-27T21:36:06,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:06,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. as already flushing 2024-11-27T21:36:06,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:06,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:06,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:06,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:06,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-27T21:36:06,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-27T21:36:06,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:06,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. as already flushing 2024-11-27T21:36:06,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:06,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:06,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:06,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:06,870 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=213, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/d33616f433904988a4895f47b43ad6b9 2024-11-27T21:36:06,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:06,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743426863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:06,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:06,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:06,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:06,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743426864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:06,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:06,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:06,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:06,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743426864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:06,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:06,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:06,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:06,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:06,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:06,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:06,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743426864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:06,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:06,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:06,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:06,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/e0a5f795f0e14a98a8dac55784a11356 is 50, key is test_row_0/B:col10/1732743366450/Put/seqid=0 2024-11-27T21:36:06,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:06,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743426873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:06,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:06,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:06,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:06,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742234_1410 (size=12151) 2024-11-27T21:36:07,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-27T21:36:07,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:07,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. as already flushing 2024-11-27T21:36:07,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:07,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:07,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:07,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:07,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-27T21:36:07,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-27T21:36:07,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:07,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. as already flushing 2024-11-27T21:36:07,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:07,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:07,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:07,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:07,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:07,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743427183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:07,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:07,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:07,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:07,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:07,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743427183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:07,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:07,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:07,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:07,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:07,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743427184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:07,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:07,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:07,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:07,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:07,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743427184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:07,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:07,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:07,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:07,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:07,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743427184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:07,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:07,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:07,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:07,282 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/e0a5f795f0e14a98a8dac55784a11356 2024-11-27T21:36:07,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/c24c6ff93e314371a8e58530eb361d43 is 50, key is test_row_0/C:col10/1732743366450/Put/seqid=0 2024-11-27T21:36:07,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742235_1411 (size=12151) 2024-11-27T21:36:07,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-27T21:36:07,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:07,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. as already flushing 2024-11-27T21:36:07,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:07,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:07,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:07,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:07,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-27T21:36:07,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:07,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. as already flushing 2024-11-27T21:36:07,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:07,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:07,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:07,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:07,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-27T21:36:07,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-27T21:36:07,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:07,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. as already flushing 2024-11-27T21:36:07,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:07,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:07,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:07,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:07,693 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/c24c6ff93e314371a8e58530eb361d43 2024-11-27T21:36:07,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/d33616f433904988a4895f47b43ad6b9 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/d33616f433904988a4895f47b43ad6b9 2024-11-27T21:36:07,701 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/d33616f433904988a4895f47b43ad6b9, entries=250, sequenceid=213, filesize=47.3 K 2024-11-27T21:36:07,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/e0a5f795f0e14a98a8dac55784a11356 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/e0a5f795f0e14a98a8dac55784a11356 2024-11-27T21:36:07,705 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/e0a5f795f0e14a98a8dac55784a11356, entries=150, sequenceid=213, filesize=11.9 K 2024-11-27T21:36:07,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:07,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:07,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743427694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:07,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743427694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:07,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/c24c6ff93e314371a8e58530eb361d43 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/c24c6ff93e314371a8e58530eb361d43 2024-11-27T21:36:07,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:07,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743427694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:07,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:07,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743427703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:07,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:07,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743427703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:07,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:07,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:07,709 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/c24c6ff93e314371a8e58530eb361d43, entries=150, sequenceid=213, filesize=11.9 K 2024-11-27T21:36:07,710 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for f9b9357dca19fe90b610172c1465ef7c in 1260ms, sequenceid=213, compaction requested=true 2024-11-27T21:36:07,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f9b9357dca19fe90b610172c1465ef7c: 2024-11-27T21:36:07,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f9b9357dca19fe90b610172c1465ef7c:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:36:07,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:07,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f9b9357dca19fe90b610172c1465ef7c:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:36:07,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:07,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f9b9357dca19fe90b610172c1465ef7c:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:36:07,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:36:07,710 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:07,710 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:07,711 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111009 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:07,711 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:07,711 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): f9b9357dca19fe90b610172c1465ef7c/A is initiating minor compaction (all files) 2024-11-27T21:36:07,711 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f9b9357dca19fe90b610172c1465ef7c/A in TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:07,711 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): f9b9357dca19fe90b610172c1465ef7c/B is initiating minor compaction (all files) 2024-11-27T21:36:07,711 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f9b9357dca19fe90b610172c1465ef7c/B in TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:07,711 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/1dbe25d1aeef479d9ff1f119d3486c11, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/ea919ac1eaa840ad9881fa5405a5b018, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/d33616f433904988a4895f47b43ad6b9] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp, totalSize=108.4 K 2024-11-27T21:36:07,711 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:07,711 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. files: [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/1dbe25d1aeef479d9ff1f119d3486c11, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/ea919ac1eaa840ad9881fa5405a5b018, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/d33616f433904988a4895f47b43ad6b9] 2024-11-27T21:36:07,711 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/13bd8897c45741d79707d3e4bc4eb1d9, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/9844b9e7749c4b4e896da6a08a09163f, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/e0a5f795f0e14a98a8dac55784a11356] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp, totalSize=36.0 K 2024-11-27T21:36:07,711 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1dbe25d1aeef479d9ff1f119d3486c11, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732743363917 2024-11-27T21:36:07,711 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 13bd8897c45741d79707d3e4bc4eb1d9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732743363917 2024-11-27T21:36:07,712 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 9844b9e7749c4b4e896da6a08a09163f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732743364086 2024-11-27T21:36:07,712 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea919ac1eaa840ad9881fa5405a5b018, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732743364086 2024-11-27T21:36:07,712 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting e0a5f795f0e14a98a8dac55784a11356, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732743365281 2024-11-27T21:36:07,712 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting d33616f433904988a4895f47b43ad6b9, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732743365278 2024-11-27T21:36:07,718 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f9b9357dca19fe90b610172c1465ef7c] 2024-11-27T21:36:07,720 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f9b9357dca19fe90b610172c1465ef7c#B#compaction#350 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:07,721 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/0606ad91817c4fc88ebdec25ec928d6e is 50, key is test_row_0/B:col10/1732743366450/Put/seqid=0 2024-11-27T21:36:07,722 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411273da9b82fa3fd4d3d8c5256b02191c57e_f9b9357dca19fe90b610172c1465ef7c store=[table=TestAcidGuarantees family=A region=f9b9357dca19fe90b610172c1465ef7c] 2024-11-27T21:36:07,723 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411273da9b82fa3fd4d3d8c5256b02191c57e_f9b9357dca19fe90b610172c1465ef7c, store=[table=TestAcidGuarantees family=A region=f9b9357dca19fe90b610172c1465ef7c] 2024-11-27T21:36:07,724 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411273da9b82fa3fd4d3d8c5256b02191c57e_f9b9357dca19fe90b610172c1465ef7c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f9b9357dca19fe90b610172c1465ef7c] 2024-11-27T21:36:07,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742236_1412 (size=12663) 2024-11-27T21:36:07,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742237_1413 (size=4469) 2024-11-27T21:36:07,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-27T21:36:07,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:07,784 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing f9b9357dca19fe90b610172c1465ef7c 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-27T21:36:07,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127a10bf3b484604b6da8401d44f870b0f9_f9b9357dca19fe90b610172c1465ef7c is 50, key is test_row_0/A:col10/1732743366528/Put/seqid=0 2024-11-27T21:36:07,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742238_1414 (size=12304) 2024-11-27T21:36:07,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:36:07,800 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127a10bf3b484604b6da8401d44f870b0f9_f9b9357dca19fe90b610172c1465ef7c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a10bf3b484604b6da8401d44f870b0f9_f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:36:07,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/7d7350b36d184b6fb40491ee8f2b949c, store: [table=TestAcidGuarantees family=A region=f9b9357dca19fe90b610172c1465ef7c] 2024-11-27T21:36:07,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/7d7350b36d184b6fb40491ee8f2b949c is 175, key is test_row_0/A:col10/1732743366528/Put/seqid=0 2024-11-27T21:36:07,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742239_1415 (size=31105) 2024-11-27T21:36:08,136 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f9b9357dca19fe90b610172c1465ef7c#A#compaction#349 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:08,136 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/0606ad91817c4fc88ebdec25ec928d6e as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/0606ad91817c4fc88ebdec25ec928d6e 2024-11-27T21:36:08,136 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/c7332f0fb71c4965b60dbfe6823888b3 is 175, key is test_row_0/A:col10/1732743366450/Put/seqid=0 2024-11-27T21:36:08,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742240_1416 (size=31617) 2024-11-27T21:36:08,142 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f9b9357dca19fe90b610172c1465ef7c/B of f9b9357dca19fe90b610172c1465ef7c into 0606ad91817c4fc88ebdec25ec928d6e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:08,143 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f9b9357dca19fe90b610172c1465ef7c: 2024-11-27T21:36:08,143 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., storeName=f9b9357dca19fe90b610172c1465ef7c/B, priority=13, startTime=1732743367710; duration=0sec 2024-11-27T21:36:08,143 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:36:08,143 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f9b9357dca19fe90b610172c1465ef7c:B 2024-11-27T21:36:08,143 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:08,144 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:08,144 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): f9b9357dca19fe90b610172c1465ef7c/C is initiating minor compaction (all files) 2024-11-27T21:36:08,144 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f9b9357dca19fe90b610172c1465ef7c/C in TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:08,144 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/1f19438ae4d54c7091ad1a0c1b910892, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/9f61529dfd16420a8f5b2a7bc7818b48, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/c24c6ff93e314371a8e58530eb361d43] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp, totalSize=36.0 K 2024-11-27T21:36:08,144 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/c7332f0fb71c4965b60dbfe6823888b3 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/c7332f0fb71c4965b60dbfe6823888b3 2024-11-27T21:36:08,145 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f19438ae4d54c7091ad1a0c1b910892, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732743363917 2024-11-27T21:36:08,145 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f61529dfd16420a8f5b2a7bc7818b48, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732743364086 2024-11-27T21:36:08,145 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting c24c6ff93e314371a8e58530eb361d43, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732743365281 2024-11-27T21:36:08,149 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f9b9357dca19fe90b610172c1465ef7c/A of f9b9357dca19fe90b610172c1465ef7c into c7332f0fb71c4965b60dbfe6823888b3(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:08,149 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f9b9357dca19fe90b610172c1465ef7c: 2024-11-27T21:36:08,149 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., storeName=f9b9357dca19fe90b610172c1465ef7c/A, priority=13, startTime=1732743367710; duration=0sec 2024-11-27T21:36:08,149 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:08,149 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f9b9357dca19fe90b610172c1465ef7c:A 2024-11-27T21:36:08,153 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f9b9357dca19fe90b610172c1465ef7c#C#compaction#352 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:08,153 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/5daa316bceb249b6a37d29f116bf9872 is 50, key is test_row_0/C:col10/1732743366450/Put/seqid=0 2024-11-27T21:36:08,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742241_1417 (size=12663) 2024-11-27T21:36:08,212 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=233, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/7d7350b36d184b6fb40491ee8f2b949c 2024-11-27T21:36:08,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/350d5484e9c34a61b8c2838ac72a6b7c is 50, key is test_row_0/B:col10/1732743366528/Put/seqid=0 2024-11-27T21:36:08,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742242_1418 (size=12151) 2024-11-27T21:36:08,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-27T21:36:08,562 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/5daa316bceb249b6a37d29f116bf9872 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/5daa316bceb249b6a37d29f116bf9872 2024-11-27T21:36:08,569 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f9b9357dca19fe90b610172c1465ef7c/C of f9b9357dca19fe90b610172c1465ef7c into 5daa316bceb249b6a37d29f116bf9872(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:08,569 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f9b9357dca19fe90b610172c1465ef7c: 2024-11-27T21:36:08,569 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., storeName=f9b9357dca19fe90b610172c1465ef7c/C, priority=13, startTime=1732743367710; duration=0sec 2024-11-27T21:36:08,569 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:08,569 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f9b9357dca19fe90b610172c1465ef7c:C 2024-11-27T21:36:08,623 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/350d5484e9c34a61b8c2838ac72a6b7c 2024-11-27T21:36:08,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/a02ddcad3e8643d490253832ed54d1ed is 50, key is test_row_0/C:col10/1732743366528/Put/seqid=0 2024-11-27T21:36:08,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742243_1419 (size=12151) 2024-11-27T21:36:08,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:36:08,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. as already flushing 2024-11-27T21:36:08,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:08,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743428737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:08,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:08,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:08,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:08,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:08,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743428743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:08,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:08,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:08,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:08,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:08,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743428744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:08,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:08,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:08,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:08,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743428744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:08,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:08,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:08,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:08,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743428745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:08,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:08,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:08,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:08,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:08,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:08,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:08,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743428853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:08,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:08,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:08,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743428854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:08,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:08,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:08,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:08,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:08,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:08,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:08,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743428863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:08,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:08,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:08,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:08,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743428864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:08,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:08,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:08,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743428864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:08,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:08,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:08,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:08,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:08,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:08,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:09,035 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/a02ddcad3e8643d490253832ed54d1ed 2024-11-27T21:36:09,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/7d7350b36d184b6fb40491ee8f2b949c as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/7d7350b36d184b6fb40491ee8f2b949c 2024-11-27T21:36:09,043 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/7d7350b36d184b6fb40491ee8f2b949c, entries=150, sequenceid=233, filesize=30.4 K 2024-11-27T21:36:09,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/350d5484e9c34a61b8c2838ac72a6b7c as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/350d5484e9c34a61b8c2838ac72a6b7c 2024-11-27T21:36:09,052 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/350d5484e9c34a61b8c2838ac72a6b7c, entries=150, sequenceid=233, filesize=11.9 K 2024-11-27T21:36:09,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/a02ddcad3e8643d490253832ed54d1ed as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/a02ddcad3e8643d490253832ed54d1ed 2024-11-27T21:36:09,057 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/a02ddcad3e8643d490253832ed54d1ed, entries=150, sequenceid=233, filesize=11.9 K 2024-11-27T21:36:09,058 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for f9b9357dca19fe90b610172c1465ef7c in 1275ms, sequenceid=233, compaction requested=false 2024-11-27T21:36:09,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for f9b9357dca19fe90b610172c1465ef7c: 2024-11-27T21:36:09,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:09,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-27T21:36:09,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-27T21:36:09,061 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-27T21:36:09,061 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6530 sec 2024-11-27T21:36:09,064 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 2.6580 sec 2024-11-27T21:36:09,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:36:09,066 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:36:09,066 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f9b9357dca19fe90b610172c1465ef7c 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-27T21:36:09,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127f6d84f852c674785a6c1ab47f09f7c7b_f9b9357dca19fe90b610172c1465ef7c is 50, key is test_row_0/A:col10/1732743369065/Put/seqid=0 2024-11-27T21:36:09,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742244_1420 (size=14794) 2024-11-27T21:36:09,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:09,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743429096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:09,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:09,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:09,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:09,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:09,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743429099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:09,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:09,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:09,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:09,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:09,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743429103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:09,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:09,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:09,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:09,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:09,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743429104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:09,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:09,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:09,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:09,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:09,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743429105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:09,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:09,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:09,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:09,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:09,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743429213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:09,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:09,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:09,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743429213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:09,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:09,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:09,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:09,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:09,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:09,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:09,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743429223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:09,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:09,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:09,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743429223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:09,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:09,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:09,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:09,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:09,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743429224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:09,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:09,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:09,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:09,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:09,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:09,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:09,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743429433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:09,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:09,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743429433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:09,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:09,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:09,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:09,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:09,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:09,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:09,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:09,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743429443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:09,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:09,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:09,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:09,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743429444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:09,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:09,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:09,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:09,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:09,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743429444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:09,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:09,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:09,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:09,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:09,479 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:36:09,483 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127f6d84f852c674785a6c1ab47f09f7c7b_f9b9357dca19fe90b610172c1465ef7c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f6d84f852c674785a6c1ab47f09f7c7b_f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:36:09,483 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/0b68645d8b9d4b89b74e07b8bc92dd1a, store: [table=TestAcidGuarantees family=A region=f9b9357dca19fe90b610172c1465ef7c] 2024-11-27T21:36:09,484 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/0b68645d8b9d4b89b74e07b8bc92dd1a is 175, key is test_row_0/A:col10/1732743369065/Put/seqid=0 2024-11-27T21:36:09,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742245_1421 (size=39749) 2024-11-27T21:36:09,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:09,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743429743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:09,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:09,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:09,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:09,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743429743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:09,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:09,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:09,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:09,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:09,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:09,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743429753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:09,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:09,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:09,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:09,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743429753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:09,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:09,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:09,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:09,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:09,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:09,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743429764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:09,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:09,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:09,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:09,888 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=254, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/0b68645d8b9d4b89b74e07b8bc92dd1a 2024-11-27T21:36:09,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/a22fb9abd30c4c11b7ce844618e28c79 is 50, key is test_row_0/B:col10/1732743369065/Put/seqid=0 2024-11-27T21:36:09,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742246_1422 (size=12151) 2024-11-27T21:36:10,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:10,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743430263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:10,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:10,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:10,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743430263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:10,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:10,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:10,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:10,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:10,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743430263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:10,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:10,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:10,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:10,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:10,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743430264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:10,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:10,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:10,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:10,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:10,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:10,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:10,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743430273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:10,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:10,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:10,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:10,299 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/a22fb9abd30c4c11b7ce844618e28c79 2024-11-27T21:36:10,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/6e5648c28fd94297a0c3d4f14c9f463f is 50, key is test_row_0/C:col10/1732743369065/Put/seqid=0 2024-11-27T21:36:10,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742247_1423 (size=12151) 2024-11-27T21:36:10,310 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/6e5648c28fd94297a0c3d4f14c9f463f 2024-11-27T21:36:10,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/0b68645d8b9d4b89b74e07b8bc92dd1a as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/0b68645d8b9d4b89b74e07b8bc92dd1a 2024-11-27T21:36:10,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/0b68645d8b9d4b89b74e07b8bc92dd1a, entries=200, sequenceid=254, filesize=38.8 K 2024-11-27T21:36:10,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/a22fb9abd30c4c11b7ce844618e28c79 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/a22fb9abd30c4c11b7ce844618e28c79 2024-11-27T21:36:10,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/a22fb9abd30c4c11b7ce844618e28c79, entries=150, sequenceid=254, filesize=11.9 K 2024-11-27T21:36:10,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/6e5648c28fd94297a0c3d4f14c9f463f as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/6e5648c28fd94297a0c3d4f14c9f463f 2024-11-27T21:36:10,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/6e5648c28fd94297a0c3d4f14c9f463f, entries=150, sequenceid=254, filesize=11.9 K 2024-11-27T21:36:10,327 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for f9b9357dca19fe90b610172c1465ef7c in 1261ms, sequenceid=254, compaction requested=true 2024-11-27T21:36:10,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f9b9357dca19fe90b610172c1465ef7c: 2024-11-27T21:36:10,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f9b9357dca19fe90b610172c1465ef7c:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:36:10,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:10,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f9b9357dca19fe90b610172c1465ef7c:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:36:10,328 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:10,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:10,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f9b9357dca19fe90b610172c1465ef7c:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:36:10,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:36:10,328 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:10,328 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:10,328 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): f9b9357dca19fe90b610172c1465ef7c/B is initiating minor compaction (all files) 2024-11-27T21:36:10,328 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f9b9357dca19fe90b610172c1465ef7c/B in TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:10,329 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/0606ad91817c4fc88ebdec25ec928d6e, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/350d5484e9c34a61b8c2838ac72a6b7c, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/a22fb9abd30c4c11b7ce844618e28c79] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp, totalSize=36.1 K 2024-11-27T21:36:10,329 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102471 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:10,329 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): f9b9357dca19fe90b610172c1465ef7c/A is initiating minor compaction (all files) 2024-11-27T21:36:10,329 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f9b9357dca19fe90b610172c1465ef7c/A in TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:10,329 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/c7332f0fb71c4965b60dbfe6823888b3, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/7d7350b36d184b6fb40491ee8f2b949c, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/0b68645d8b9d4b89b74e07b8bc92dd1a] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp, totalSize=100.1 K 2024-11-27T21:36:10,329 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:10,329 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. files: [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/c7332f0fb71c4965b60dbfe6823888b3, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/7d7350b36d184b6fb40491ee8f2b949c, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/0b68645d8b9d4b89b74e07b8bc92dd1a] 2024-11-27T21:36:10,331 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 0606ad91817c4fc88ebdec25ec928d6e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732743365281 2024-11-27T21:36:10,331 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7332f0fb71c4965b60dbfe6823888b3, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732743365281 2024-11-27T21:36:10,331 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d7350b36d184b6fb40491ee8f2b949c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732743366515 2024-11-27T21:36:10,331 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 350d5484e9c34a61b8c2838ac72a6b7c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732743366515 2024-11-27T21:36:10,331 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b68645d8b9d4b89b74e07b8bc92dd1a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732743368742 2024-11-27T21:36:10,331 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting a22fb9abd30c4c11b7ce844618e28c79, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732743368742 2024-11-27T21:36:10,336 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f9b9357dca19fe90b610172c1465ef7c] 2024-11-27T21:36:10,337 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f9b9357dca19fe90b610172c1465ef7c#B#compaction#358 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:10,338 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127ea4cac443ad34a1fb8dd445a3e0d25ad_f9b9357dca19fe90b610172c1465ef7c store=[table=TestAcidGuarantees family=A region=f9b9357dca19fe90b610172c1465ef7c] 2024-11-27T21:36:10,338 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/31dd7b665c904118bc17d37f6a11a28d is 50, key is test_row_0/B:col10/1732743369065/Put/seqid=0 2024-11-27T21:36:10,340 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127ea4cac443ad34a1fb8dd445a3e0d25ad_f9b9357dca19fe90b610172c1465ef7c, store=[table=TestAcidGuarantees family=A region=f9b9357dca19fe90b610172c1465ef7c] 2024-11-27T21:36:10,340 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127ea4cac443ad34a1fb8dd445a3e0d25ad_f9b9357dca19fe90b610172c1465ef7c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f9b9357dca19fe90b610172c1465ef7c] 2024-11-27T21:36:10,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742248_1424 (size=4469) 2024-11-27T21:36:10,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742249_1425 (size=12765) 2024-11-27T21:36:10,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-27T21:36:10,563 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:36:10,564 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:36:10,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-27T21:36:10,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-27T21:36:10,566 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:36:10,566 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:36:10,567 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:36:10,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-27T21:36:10,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-27T21:36:10,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:10,719 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing f9b9357dca19fe90b610172c1465ef7c 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-27T21:36:10,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d0b5e56b9cec4fff8fc2212dd8ba6606_f9b9357dca19fe90b610172c1465ef7c is 50, key is test_row_0/A:col10/1732743369103/Put/seqid=0 2024-11-27T21:36:10,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742250_1426 (size=12454) 2024-11-27T21:36:10,747 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f9b9357dca19fe90b610172c1465ef7c#A#compaction#359 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:10,747 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/aef9f288ef2641758ef8bc6b85177c25 is 175, key is test_row_0/A:col10/1732743369065/Put/seqid=0 2024-11-27T21:36:10,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742251_1427 (size=31719) 2024-11-27T21:36:10,759 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/31dd7b665c904118bc17d37f6a11a28d as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/31dd7b665c904118bc17d37f6a11a28d 2024-11-27T21:36:10,763 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f9b9357dca19fe90b610172c1465ef7c/B of f9b9357dca19fe90b610172c1465ef7c into 31dd7b665c904118bc17d37f6a11a28d(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:10,763 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f9b9357dca19fe90b610172c1465ef7c: 2024-11-27T21:36:10,764 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., storeName=f9b9357dca19fe90b610172c1465ef7c/B, priority=13, startTime=1732743370328; duration=0sec 2024-11-27T21:36:10,764 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:36:10,764 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f9b9357dca19fe90b610172c1465ef7c:B 2024-11-27T21:36:10,764 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:10,765 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:10,765 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): f9b9357dca19fe90b610172c1465ef7c/C is initiating minor compaction (all files) 2024-11-27T21:36:10,765 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f9b9357dca19fe90b610172c1465ef7c/C in TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:10,765 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/5daa316bceb249b6a37d29f116bf9872, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/a02ddcad3e8643d490253832ed54d1ed, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/6e5648c28fd94297a0c3d4f14c9f463f] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp, totalSize=36.1 K 2024-11-27T21:36:10,765 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 5daa316bceb249b6a37d29f116bf9872, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732743365281 2024-11-27T21:36:10,765 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting a02ddcad3e8643d490253832ed54d1ed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732743366515 2024-11-27T21:36:10,766 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e5648c28fd94297a0c3d4f14c9f463f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732743368742 2024-11-27T21:36:10,772 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f9b9357dca19fe90b610172c1465ef7c#C#compaction#361 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:10,772 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/a100b3bd77904373b17eb8bbca026900 is 50, key is test_row_0/C:col10/1732743369065/Put/seqid=0 2024-11-27T21:36:10,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742252_1428 (size=12765) 2024-11-27T21:36:10,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-27T21:36:11,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:36:11,133 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d0b5e56b9cec4fff8fc2212dd8ba6606_f9b9357dca19fe90b610172c1465ef7c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d0b5e56b9cec4fff8fc2212dd8ba6606_f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:36:11,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/0e5a1572df1f41169e81fc28f242b3e6, store: [table=TestAcidGuarantees family=A region=f9b9357dca19fe90b610172c1465ef7c] 2024-11-27T21:36:11,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/0e5a1572df1f41169e81fc28f242b3e6 is 175, key is test_row_0/A:col10/1732743369103/Put/seqid=0 2024-11-27T21:36:11,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742253_1429 (size=31255) 2024-11-27T21:36:11,158 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/aef9f288ef2641758ef8bc6b85177c25 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/aef9f288ef2641758ef8bc6b85177c25 2024-11-27T21:36:11,164 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f9b9357dca19fe90b610172c1465ef7c/A of f9b9357dca19fe90b610172c1465ef7c into aef9f288ef2641758ef8bc6b85177c25(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:11,164 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f9b9357dca19fe90b610172c1465ef7c: 2024-11-27T21:36:11,164 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., storeName=f9b9357dca19fe90b610172c1465ef7c/A, priority=13, startTime=1732743370327; duration=0sec 2024-11-27T21:36:11,164 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:11,164 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f9b9357dca19fe90b610172c1465ef7c:A 2024-11-27T21:36:11,179 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/a100b3bd77904373b17eb8bbca026900 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/a100b3bd77904373b17eb8bbca026900 2024-11-27T21:36:11,184 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f9b9357dca19fe90b610172c1465ef7c/C of f9b9357dca19fe90b610172c1465ef7c into a100b3bd77904373b17eb8bbca026900(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:11,184 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f9b9357dca19fe90b610172c1465ef7c: 2024-11-27T21:36:11,184 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., storeName=f9b9357dca19fe90b610172c1465ef7c/C, priority=13, startTime=1732743370328; duration=0sec 2024-11-27T21:36:11,184 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:11,184 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f9b9357dca19fe90b610172c1465ef7c:C 2024-11-27T21:36:11,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-27T21:36:11,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:36:11,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. as already flushing 2024-11-27T21:36:11,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:11,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743431305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:11,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:11,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743431313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:11,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:11,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:11,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:11,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:11,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:11,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:11,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:11,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743431315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:11,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:11,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:11,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:11,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:11,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743431316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:11,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:11,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:11,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:11,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:11,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743431317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:11,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:11,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:11,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:11,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:11,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743431423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:11,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:11,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:11,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743431423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:11,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:11,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:11,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:11,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:11,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:11,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:11,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743431434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:11,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:11,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:11,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:11,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743431434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:11,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:11,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:11,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:11,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:11,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:11,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743431434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:11,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:11,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:11,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:11,540 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=272, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/0e5a1572df1f41169e81fc28f242b3e6 2024-11-27T21:36:11,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/698b68b181ab4878a19d5a867d3864e7 is 50, key is test_row_0/B:col10/1732743369103/Put/seqid=0 2024-11-27T21:36:11,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742254_1430 (size=12301) 2024-11-27T21:36:11,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:11,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743431633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:11,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:11,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:11,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743431633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:11,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:11,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:11,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:11,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:11,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:11,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:11,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743431653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:11,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:11,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:11,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:11,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743431653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:11,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:11,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:11,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:11,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:11,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:11,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743431653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:11,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:11,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:11,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:11,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-27T21:36:11,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:11,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743431943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:11,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:11,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743431943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:11,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:11,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:11,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:11,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:11,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:11,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:11,951 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/698b68b181ab4878a19d5a867d3864e7 2024-11-27T21:36:11,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/a2fa3fc4786f4c0e9bf2798221617ea9 is 50, key is test_row_0/C:col10/1732743369103/Put/seqid=0 2024-11-27T21:36:11,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742255_1431 (size=12301) 2024-11-27T21:36:11,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:11,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743431963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:11,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:11,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:11,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:11,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743431963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:11,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:11,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:11,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:11,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:11,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:11,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743431963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:11,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:11,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:11,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:12,362 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/a2fa3fc4786f4c0e9bf2798221617ea9 2024-11-27T21:36:12,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/0e5a1572df1f41169e81fc28f242b3e6 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/0e5a1572df1f41169e81fc28f242b3e6 2024-11-27T21:36:12,370 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/0e5a1572df1f41169e81fc28f242b3e6, entries=150, sequenceid=272, filesize=30.5 K 2024-11-27T21:36:12,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/698b68b181ab4878a19d5a867d3864e7 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/698b68b181ab4878a19d5a867d3864e7 2024-11-27T21:36:12,374 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/698b68b181ab4878a19d5a867d3864e7, entries=150, sequenceid=272, filesize=12.0 K 2024-11-27T21:36:12,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/a2fa3fc4786f4c0e9bf2798221617ea9 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/a2fa3fc4786f4c0e9bf2798221617ea9 2024-11-27T21:36:12,378 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/a2fa3fc4786f4c0e9bf2798221617ea9, entries=150, sequenceid=272, filesize=12.0 K 2024-11-27T21:36:12,379 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for f9b9357dca19fe90b610172c1465ef7c in 1660ms, sequenceid=272, compaction requested=false 2024-11-27T21:36:12,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for f9b9357dca19fe90b610172c1465ef7c: 2024-11-27T21:36:12,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:12,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-27T21:36:12,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-27T21:36:12,382 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-27T21:36:12,382 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8150 sec 2024-11-27T21:36:12,384 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.8190 sec 2024-11-27T21:36:12,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:36:12,458 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:36:12,458 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f9b9357dca19fe90b610172c1465ef7c 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-27T21:36:12,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411272a67830b56fa4b62b84101bc706659cd_f9b9357dca19fe90b610172c1465ef7c is 50, key is test_row_0/A:col10/1732743371315/Put/seqid=0 2024-11-27T21:36:12,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742256_1432 (size=14994) 2024-11-27T21:36:12,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:12,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743432492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:12,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:12,506 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:12,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743432493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:12,506 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:12,506 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:12,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:12,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:12,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:12,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743432494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:12,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:12,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:12,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:12,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743432495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:12,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:12,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:12,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:12,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:12,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:12,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:12,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743432505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:12,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:12,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:12,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:12,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:12,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743432613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:12,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:12,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:12,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:12,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:12,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743432614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:12,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:12,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:12,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:12,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:12,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743432614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:12,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:12,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:12,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:12,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743432614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:12,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:12,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:12,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:12,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:12,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:12,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743432614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:12,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:12,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:12,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:12,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-27T21:36:12,714 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:36:12,714 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:36:12,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-27T21:36:12,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-27T21:36:12,716 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:36:12,717 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:36:12,717 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:36:12,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-27T21:36:12,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:12,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743432823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:12,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:12,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:12,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743432824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:12,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:12,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:12,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:12,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:12,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:12,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:12,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743432824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:12,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:12,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:12,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:12,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:12,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743432824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:12,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:12,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:12,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:12,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:12,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743432824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:12,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:12,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:12,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:12,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-27T21:36:12,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:12,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. as already flushing 2024-11-27T21:36:12,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:12,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:12,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:12,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:12,871 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:36:12,874 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411272a67830b56fa4b62b84101bc706659cd_f9b9357dca19fe90b610172c1465ef7c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411272a67830b56fa4b62b84101bc706659cd_f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:36:12,875 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/bc768e865b1b45efb4fbea02d214a033, store: [table=TestAcidGuarantees family=A region=f9b9357dca19fe90b610172c1465ef7c] 2024-11-27T21:36:12,876 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/bc768e865b1b45efb4fbea02d214a033 is 175, key is test_row_0/A:col10/1732743371315/Put/seqid=0 2024-11-27T21:36:12,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742257_1433 (size=39949) 2024-11-27T21:36:13,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-27T21:36:13,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:13,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. as already flushing 2024-11-27T21:36:13,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:13,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:13,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:13,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:13,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-27T21:36:13,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:13,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743433143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:13,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:13,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743433144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:13,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:13,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:13,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:13,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:13,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:13,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743433144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:13,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:13,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:13,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:13,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:13,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743433144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:13,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:13,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:13,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:13,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:13,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:13,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743433144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:13,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:13,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:13,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:13,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:13,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-27T21:36:13,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:13,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. as already flushing 2024-11-27T21:36:13,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:13,175 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:13,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:13,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:13,280 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=295, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/bc768e865b1b45efb4fbea02d214a033 2024-11-27T21:36:13,287 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/049866951b174724b8d414c749f0063f is 50, key is test_row_0/B:col10/1732743371315/Put/seqid=0 2024-11-27T21:36:13,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742258_1434 (size=12301) 2024-11-27T21:36:13,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-27T21:36:13,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:13,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. as already flushing 2024-11-27T21:36:13,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:13,328 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:13,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:13,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:13,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-27T21:36:13,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-27T21:36:13,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:13,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. as already flushing 2024-11-27T21:36:13,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:13,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:13,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:13,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:13,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-27T21:36:13,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:13,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. as already flushing 2024-11-27T21:36:13,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:13,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:13,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:13,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:13,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:13,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743433663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:13,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:13,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:13,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:13,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:13,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743433663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:13,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:13,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:13,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:13,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:13,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743433663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:13,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:13,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:13,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:13,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:13,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743433663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:13,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:13,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:13,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:13,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:13,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743433663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:13,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:13,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:13,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:13,690 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/049866951b174724b8d414c749f0063f 2024-11-27T21:36:13,697 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/36598795ee5b40c9a8c252d1cb3b2d0a is 50, key is test_row_0/C:col10/1732743371315/Put/seqid=0 2024-11-27T21:36:13,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742259_1435 (size=12301) 2024-11-27T21:36:13,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-27T21:36:13,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:13,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. as already flushing 2024-11-27T21:36:13,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:13,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:13,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:13,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:13,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-27T21:36:13,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-27T21:36:13,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:13,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. as already flushing 2024-11-27T21:36:13,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:13,942 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:13,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:13,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:14,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-27T21:36:14,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:14,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. as already flushing 2024-11-27T21:36:14,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:14,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:14,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:14,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:14,101 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/36598795ee5b40c9a8c252d1cb3b2d0a 2024-11-27T21:36:14,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/bc768e865b1b45efb4fbea02d214a033 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/bc768e865b1b45efb4fbea02d214a033 2024-11-27T21:36:14,113 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/bc768e865b1b45efb4fbea02d214a033, entries=200, sequenceid=295, filesize=39.0 K 2024-11-27T21:36:14,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/049866951b174724b8d414c749f0063f as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/049866951b174724b8d414c749f0063f 2024-11-27T21:36:14,117 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/049866951b174724b8d414c749f0063f, entries=150, sequenceid=295, filesize=12.0 K 2024-11-27T21:36:14,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/36598795ee5b40c9a8c252d1cb3b2d0a as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/36598795ee5b40c9a8c252d1cb3b2d0a 2024-11-27T21:36:14,124 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/36598795ee5b40c9a8c252d1cb3b2d0a, entries=150, sequenceid=295, filesize=12.0 K 2024-11-27T21:36:14,125 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for f9b9357dca19fe90b610172c1465ef7c in 1667ms, sequenceid=295, compaction requested=true 2024-11-27T21:36:14,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f9b9357dca19fe90b610172c1465ef7c: 2024-11-27T21:36:14,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f9b9357dca19fe90b610172c1465ef7c:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:36:14,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:14,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f9b9357dca19fe90b610172c1465ef7c:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:36:14,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:14,126 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:14,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f9b9357dca19fe90b610172c1465ef7c:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:36:14,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:36:14,126 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:14,126 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:14,126 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102923 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:14,126 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): f9b9357dca19fe90b610172c1465ef7c/A is initiating minor compaction (all files) 2024-11-27T21:36:14,126 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): f9b9357dca19fe90b610172c1465ef7c/B is initiating minor compaction (all files) 2024-11-27T21:36:14,127 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f9b9357dca19fe90b610172c1465ef7c/A in TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:14,127 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f9b9357dca19fe90b610172c1465ef7c/B in TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:14,127 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/aef9f288ef2641758ef8bc6b85177c25, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/0e5a1572df1f41169e81fc28f242b3e6, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/bc768e865b1b45efb4fbea02d214a033] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp, totalSize=100.5 K 2024-11-27T21:36:14,127 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/31dd7b665c904118bc17d37f6a11a28d, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/698b68b181ab4878a19d5a867d3864e7, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/049866951b174724b8d414c749f0063f] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp, totalSize=36.5 K 2024-11-27T21:36:14,127 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:14,127 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. files: [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/aef9f288ef2641758ef8bc6b85177c25, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/0e5a1572df1f41169e81fc28f242b3e6, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/bc768e865b1b45efb4fbea02d214a033] 2024-11-27T21:36:14,127 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 31dd7b665c904118bc17d37f6a11a28d, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732743368742 2024-11-27T21:36:14,127 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting aef9f288ef2641758ef8bc6b85177c25, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732743368742 2024-11-27T21:36:14,127 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 698b68b181ab4878a19d5a867d3864e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732743369102 2024-11-27T21:36:14,127 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e5a1572df1f41169e81fc28f242b3e6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732743369102 2024-11-27T21:36:14,128 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 049866951b174724b8d414c749f0063f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732743371315 2024-11-27T21:36:14,128 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc768e865b1b45efb4fbea02d214a033, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732743371302 2024-11-27T21:36:14,135 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f9b9357dca19fe90b610172c1465ef7c#B#compaction#367 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:14,135 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/e8f64b1f196d48e984f5e5f662ac0902 is 50, key is test_row_0/B:col10/1732743371315/Put/seqid=0 2024-11-27T21:36:14,138 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f9b9357dca19fe90b610172c1465ef7c] 2024-11-27T21:36:14,145 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127153ecfd32fd746349acae9b1d47c30bf_f9b9357dca19fe90b610172c1465ef7c store=[table=TestAcidGuarantees family=A region=f9b9357dca19fe90b610172c1465ef7c] 2024-11-27T21:36:14,147 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127153ecfd32fd746349acae9b1d47c30bf_f9b9357dca19fe90b610172c1465ef7c, store=[table=TestAcidGuarantees family=A region=f9b9357dca19fe90b610172c1465ef7c] 2024-11-27T21:36:14,147 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127153ecfd32fd746349acae9b1d47c30bf_f9b9357dca19fe90b610172c1465ef7c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f9b9357dca19fe90b610172c1465ef7c] 2024-11-27T21:36:14,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742260_1436 (size=13017) 2024-11-27T21:36:14,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742261_1437 (size=4469) 2024-11-27T21:36:14,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-27T21:36:14,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:14,249 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing f9b9357dca19fe90b610172c1465ef7c 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-27T21:36:14,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411279bde3d40c3b343fea5cdd91052c12cb7_f9b9357dca19fe90b610172c1465ef7c is 50, key is test_row_0/A:col10/1732743372490/Put/seqid=0 2024-11-27T21:36:14,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742262_1438 (size=12454) 2024-11-27T21:36:14,558 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/e8f64b1f196d48e984f5e5f662ac0902 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/e8f64b1f196d48e984f5e5f662ac0902 2024-11-27T21:36:14,560 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f9b9357dca19fe90b610172c1465ef7c#A#compaction#368 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:14,561 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/4e7145f5ef04430b858cffe8de9edb04 is 175, key is test_row_0/A:col10/1732743371315/Put/seqid=0 2024-11-27T21:36:14,562 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f9b9357dca19fe90b610172c1465ef7c/B of f9b9357dca19fe90b610172c1465ef7c into e8f64b1f196d48e984f5e5f662ac0902(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:14,563 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f9b9357dca19fe90b610172c1465ef7c: 2024-11-27T21:36:14,563 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., storeName=f9b9357dca19fe90b610172c1465ef7c/B, priority=13, startTime=1732743374126; duration=0sec 2024-11-27T21:36:14,563 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:36:14,563 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f9b9357dca19fe90b610172c1465ef7c:B 2024-11-27T21:36:14,563 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:14,564 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:14,564 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): f9b9357dca19fe90b610172c1465ef7c/C is initiating minor compaction (all files) 2024-11-27T21:36:14,564 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f9b9357dca19fe90b610172c1465ef7c/C in TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:14,564 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/a100b3bd77904373b17eb8bbca026900, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/a2fa3fc4786f4c0e9bf2798221617ea9, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/36598795ee5b40c9a8c252d1cb3b2d0a] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp, totalSize=36.5 K 2024-11-27T21:36:14,564 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting a100b3bd77904373b17eb8bbca026900, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732743368742 2024-11-27T21:36:14,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742263_1439 (size=31971) 2024-11-27T21:36:14,565 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting a2fa3fc4786f4c0e9bf2798221617ea9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732743369102 2024-11-27T21:36:14,565 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 36598795ee5b40c9a8c252d1cb3b2d0a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732743371315 2024-11-27T21:36:14,571 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/4e7145f5ef04430b858cffe8de9edb04 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/4e7145f5ef04430b858cffe8de9edb04 2024-11-27T21:36:14,576 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f9b9357dca19fe90b610172c1465ef7c#C#compaction#370 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:14,576 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f9b9357dca19fe90b610172c1465ef7c/A of f9b9357dca19fe90b610172c1465ef7c into 4e7145f5ef04430b858cffe8de9edb04(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:14,576 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f9b9357dca19fe90b610172c1465ef7c: 2024-11-27T21:36:14,576 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., storeName=f9b9357dca19fe90b610172c1465ef7c/A, priority=13, startTime=1732743374125; duration=0sec 2024-11-27T21:36:14,576 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/eeb46a486e304d5e848e0f9c74550a37 is 50, key is test_row_0/C:col10/1732743371315/Put/seqid=0 2024-11-27T21:36:14,577 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:14,577 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f9b9357dca19fe90b610172c1465ef7c:A 2024-11-27T21:36:14,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742264_1440 (size=13017) 2024-11-27T21:36:14,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:36:14,664 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411279bde3d40c3b343fea5cdd91052c12cb7_f9b9357dca19fe90b610172c1465ef7c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279bde3d40c3b343fea5cdd91052c12cb7_f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:36:14,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/d4443b02c62c43229c61a97f051a763c, store: [table=TestAcidGuarantees family=A region=f9b9357dca19fe90b610172c1465ef7c] 2024-11-27T21:36:14,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/d4443b02c62c43229c61a97f051a763c is 175, key is test_row_0/A:col10/1732743372490/Put/seqid=0 2024-11-27T21:36:14,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742265_1441 (size=31255) 2024-11-27T21:36:14,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:36:14,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. as already flushing 2024-11-27T21:36:14,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:14,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743434705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:14,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:14,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:14,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:14,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:14,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743434706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:14,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:14,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:14,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:14,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:14,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743434710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:14,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:14,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:14,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:14,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:14,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743434711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:14,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:14,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:14,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:14,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:14,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743434717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:14,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:14,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:14,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:14,776 INFO [Thread-1605 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1605. 2024-11-27T21:36:14,776 DEBUG [Thread-1605 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:36:14,776 DEBUG [Thread-1605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:14,777 DEBUG [Thread-1605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:14,777 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:36:14,779 INFO [Thread-1603 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1603. 2024-11-27T21:36:14,779 DEBUG [Thread-1603 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:36:14,779 DEBUG [Thread-1603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:14,779 DEBUG [Thread-1603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:14,779 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:36:14,781 INFO [Thread-1607 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1607. 2024-11-27T21:36:14,781 DEBUG [Thread-1607 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:36:14,781 DEBUG [Thread-1607 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:14,781 DEBUG [Thread-1607 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:14,781 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:36:14,788 INFO [Thread-1601 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1601. 2024-11-27T21:36:14,788 DEBUG [Thread-1601 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:36:14,788 DEBUG [Thread-1601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:14,788 INFO [Thread-1599 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1599. 2024-11-27T21:36:14,788 DEBUG [Thread-1599 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:36:14,789 DEBUG [Thread-1601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:14,789 DEBUG [Thread-1599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:14,789 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:36:14,789 DEBUG [Thread-1599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:14,789 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:36:14,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:14,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:14,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:14,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743434823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:14,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743434823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:14,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743434823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:14,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:14,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:14,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:14,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:14,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:14,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:14,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:14,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:14,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:14,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:14,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743434834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:14,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:14,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743434834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:14,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:14,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:14,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:14,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:14,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:14,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:14,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-27T21:36:14,990 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/eeb46a486e304d5e848e0f9c74550a37 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/eeb46a486e304d5e848e0f9c74550a37 2024-11-27T21:36:14,993 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f9b9357dca19fe90b610172c1465ef7c/C of f9b9357dca19fe90b610172c1465ef7c into eeb46a486e304d5e848e0f9c74550a37(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:14,993 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f9b9357dca19fe90b610172c1465ef7c: 2024-11-27T21:36:14,994 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., storeName=f9b9357dca19fe90b610172c1465ef7c/C, priority=13, startTime=1732743374126; duration=0sec 2024-11-27T21:36:14,994 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:14,994 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f9b9357dca19fe90b610172c1465ef7c:C 2024-11-27T21:36:15,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:15,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:15,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743435033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:15,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:15,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743435033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:15,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743435034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:15,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:15,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:15,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:15,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:15,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:15,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:15,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:15,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:15,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:15,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:15,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743435044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:15,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:15,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743435044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:15,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:15,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:15,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:15,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:15,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:15,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:15,069 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=311, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/d4443b02c62c43229c61a97f051a763c 2024-11-27T21:36:15,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/bd97371522cd481e9a4b5605b27a470e is 50, key is test_row_0/B:col10/1732743372490/Put/seqid=0 2024-11-27T21:36:15,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742266_1442 (size=12301) 2024-11-27T21:36:15,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:15,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:15,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:15,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743435343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:15,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743435343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:15,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743435343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:15,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:15,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:15,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:15,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:15,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:15,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:15,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:15,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:15,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:15,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:15,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:15,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743435353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:15,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743435353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:15,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:15,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:15,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:15,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:15,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:15,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:15,479 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/bd97371522cd481e9a4b5605b27a470e 2024-11-27T21:36:15,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/4069d514a65c4a4e9f3b65b0745ff7ba is 50, key is test_row_0/C:col10/1732743372490/Put/seqid=0 2024-11-27T21:36:15,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742267_1443 (size=12301) 2024-11-27T21:36:15,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:15,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:15,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:15,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42806 deadline: 1732743435854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:15,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42776 deadline: 1732743435854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:15,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42784 deadline: 1732743435854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:15,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:15,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:15,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:15,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:15,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:15,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:15,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:15,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:15,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:15,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:15,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:15,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42762 deadline: 1732743435863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:15,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42800 deadline: 1732743435863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:15,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:15,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:15,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:15,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:15,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:15,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:15,888 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/4069d514a65c4a4e9f3b65b0745ff7ba 2024-11-27T21:36:15,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/d4443b02c62c43229c61a97f051a763c as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/d4443b02c62c43229c61a97f051a763c 2024-11-27T21:36:15,894 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/d4443b02c62c43229c61a97f051a763c, entries=150, sequenceid=311, filesize=30.5 K 2024-11-27T21:36:15,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/bd97371522cd481e9a4b5605b27a470e as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/bd97371522cd481e9a4b5605b27a470e 2024-11-27T21:36:15,897 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/bd97371522cd481e9a4b5605b27a470e, entries=150, sequenceid=311, filesize=12.0 K 2024-11-27T21:36:15,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/4069d514a65c4a4e9f3b65b0745ff7ba as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/4069d514a65c4a4e9f3b65b0745ff7ba 2024-11-27T21:36:15,900 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/4069d514a65c4a4e9f3b65b0745ff7ba, entries=150, sequenceid=311, filesize=12.0 K 2024-11-27T21:36:15,901 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for f9b9357dca19fe90b610172c1465ef7c in 1652ms, sequenceid=311, compaction requested=false 2024-11-27T21:36:15,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for f9b9357dca19fe90b610172c1465ef7c: 2024-11-27T21:36:15,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:15,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-27T21:36:15,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-27T21:36:15,903 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-27T21:36:15,903 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1850 sec 2024-11-27T21:36:15,905 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 3.1900 sec 2024-11-27T21:36:16,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:36:16,864 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:36:16,865 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f9b9357dca19fe90b610172c1465ef7c 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-27T21:36:16,865 INFO [Thread-1588 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1588. 2024-11-27T21:36:16,865 DEBUG [Thread-1588 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:36:16,865 DEBUG [Thread-1588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:16,865 DEBUG [Thread-1588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:16,865 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:36:16,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127f3b5cb37a1684db2a6365da0c6ae0994_f9b9357dca19fe90b610172c1465ef7c is 50, key is test_row_0/A:col10/1732743374709/Put/seqid=0 2024-11-27T21:36:16,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742268_1444 (size=12454) 2024-11-27T21:36:16,874 INFO [Thread-1590 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1590. 2024-11-27T21:36:16,874 INFO [Thread-1596 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1596. 2024-11-27T21:36:16,874 DEBUG [Thread-1590 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:36:16,874 DEBUG [Thread-1596 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:36:16,874 DEBUG [Thread-1590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:16,874 DEBUG [Thread-1596 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:16,874 DEBUG [Thread-1590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:16,874 DEBUG [Thread-1596 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:16,874 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:36:16,874 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:36:16,885 INFO [Thread-1594 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1594. 2024-11-27T21:36:16,885 INFO [Thread-1592 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1592. 2024-11-27T21:36:16,885 DEBUG [Thread-1594 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:36:16,885 DEBUG [Thread-1592 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:36:16,885 DEBUG [Thread-1594 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:16,885 DEBUG [Thread-1592 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:16,885 DEBUG [Thread-1592 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:16,885 DEBUG [Thread-1594 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:16,885 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:36:16,885 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:36:16,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-27T21:36:16,894 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:36:16,894 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-27T21:36:16,894 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-27T21:36:16,894 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-27T21:36:16,894 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-27T21:36:16,894 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-27T21:36:16,894 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-27T21:36:16,894 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-27T21:36:16,894 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-27T21:36:16,894 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1207 2024-11-27T21:36:16,894 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3620 rows 2024-11-27T21:36:16,894 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1208 2024-11-27T21:36:16,894 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3616 rows 2024-11-27T21:36:16,894 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1216 2024-11-27T21:36:16,894 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3635 rows 2024-11-27T21:36:16,894 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1213 2024-11-27T21:36:16,894 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3630 rows 2024-11-27T21:36:16,894 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1207 2024-11-27T21:36:16,894 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3608 rows 2024-11-27T21:36:16,894 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T21:36:16,894 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T21:36:16,894 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:16,894 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:16,894 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:36:16,895 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-27T21:36:16,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-27T21:36:16,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-27T21:36:16,898 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732743376898"}]},"ts":"1732743376898"} 2024-11-27T21:36:16,899 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-27T21:36:16,899 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-27T21:36:16,900 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T21:36:16,901 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f9b9357dca19fe90b610172c1465ef7c, UNASSIGN}] 2024-11-27T21:36:16,902 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f9b9357dca19fe90b610172c1465ef7c, UNASSIGN 2024-11-27T21:36:16,902 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=f9b9357dca19fe90b610172c1465ef7c, regionState=CLOSING, regionLocation=4585c39bed03,39097,1732743267048 2024-11-27T21:36:16,903 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f9b9357dca19fe90b610172c1465ef7c, UNASSIGN because future has completed 2024-11-27T21:36:16,903 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T21:36:16,903 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; CloseRegionProcedure f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048}] 2024-11-27T21:36:17,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-27T21:36:17,054 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(122): Close f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:36:17,055 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T21:36:17,055 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1722): Closing f9b9357dca19fe90b610172c1465ef7c, disabling compactions & flushes 2024-11-27T21:36:17,055 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:17,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-27T21:36:17,273 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:36:17,277 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127f3b5cb37a1684db2a6365da0c6ae0994_f9b9357dca19fe90b610172c1465ef7c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f3b5cb37a1684db2a6365da0c6ae0994_f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:36:17,278 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/0eb5456ddcec4f87a891156f90fd0b69, store: [table=TestAcidGuarantees family=A region=f9b9357dca19fe90b610172c1465ef7c] 2024-11-27T21:36:17,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/0eb5456ddcec4f87a891156f90fd0b69 is 175, key is test_row_0/A:col10/1732743374709/Put/seqid=0 2024-11-27T21:36:17,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742269_1445 (size=31255) 2024-11-27T21:36:17,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-27T21:36:17,682 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=335, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/0eb5456ddcec4f87a891156f90fd0b69 2024-11-27T21:36:17,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/9d66757bd7fa4640979b06caa7cbd369 is 50, key is test_row_0/B:col10/1732743374709/Put/seqid=0 2024-11-27T21:36:17,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742270_1446 (size=12301) 2024-11-27T21:36:18,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-27T21:36:18,091 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/9d66757bd7fa4640979b06caa7cbd369 2024-11-27T21:36:18,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/c3921e054d1c49d8884dc68792abde80 is 50, key is test_row_0/C:col10/1732743374709/Put/seqid=0 2024-11-27T21:36:18,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742271_1447 (size=12301) 2024-11-27T21:36:18,500 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/c3921e054d1c49d8884dc68792abde80 2024-11-27T21:36:18,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/0eb5456ddcec4f87a891156f90fd0b69 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/0eb5456ddcec4f87a891156f90fd0b69 2024-11-27T21:36:18,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/0eb5456ddcec4f87a891156f90fd0b69, entries=150, sequenceid=335, filesize=30.5 K 2024-11-27T21:36:18,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/9d66757bd7fa4640979b06caa7cbd369 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/9d66757bd7fa4640979b06caa7cbd369 2024-11-27T21:36:18,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/9d66757bd7fa4640979b06caa7cbd369, entries=150, sequenceid=335, filesize=12.0 K 2024-11-27T21:36:18,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/c3921e054d1c49d8884dc68792abde80 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/c3921e054d1c49d8884dc68792abde80 2024-11-27T21:36:18,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/c3921e054d1c49d8884dc68792abde80, entries=150, sequenceid=335, filesize=12.0 K 2024-11-27T21:36:18,515 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=26.84 KB/27480 for f9b9357dca19fe90b610172c1465ef7c in 1651ms, sequenceid=335, compaction requested=true 2024-11-27T21:36:18,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f9b9357dca19fe90b610172c1465ef7c: 2024-11-27T21:36:18,515 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:18,515 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:18,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f9b9357dca19fe90b610172c1465ef7c:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:36:18,515 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. after waiting 0 ms 2024-11-27T21:36:18,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:18,515 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. because compaction request was cancelled 2024-11-27T21:36:18,515 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:18,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f9b9357dca19fe90b610172c1465ef7c:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:36:18,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:18,515 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. because compaction request was cancelled 2024-11-27T21:36:18,515 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f9b9357dca19fe90b610172c1465ef7c:A 2024-11-27T21:36:18,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f9b9357dca19fe90b610172c1465ef7c:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:36:18,515 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f9b9357dca19fe90b610172c1465ef7c:B 2024-11-27T21:36:18,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:18,515 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. because compaction request was cancelled 2024-11-27T21:36:18,515 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f9b9357dca19fe90b610172c1465ef7c:C 2024-11-27T21:36:18,515 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(2902): Flushing f9b9357dca19fe90b610172c1465ef7c 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-27T21:36:18,519 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127f8866f3cc05d4e8d8681cc0c52fc15e5_f9b9357dca19fe90b610172c1465ef7c is 50, key is test_row_1/A:col10/1732743376883/Put/seqid=0 2024-11-27T21:36:18,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742272_1448 (size=9914) 2024-11-27T21:36:18,923 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:36:18,926 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127f8866f3cc05d4e8d8681cc0c52fc15e5_f9b9357dca19fe90b610172c1465ef7c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f8866f3cc05d4e8d8681cc0c52fc15e5_f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:36:18,927 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/54c8436e93274722a5d87489354dd043, store: [table=TestAcidGuarantees family=A region=f9b9357dca19fe90b610172c1465ef7c] 2024-11-27T21:36:18,927 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/54c8436e93274722a5d87489354dd043 is 175, key is test_row_1/A:col10/1732743376883/Put/seqid=0 2024-11-27T21:36:18,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742273_1449 (size=22561) 2024-11-27T21:36:19,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-27T21:36:19,331 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=342, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/54c8436e93274722a5d87489354dd043 2024-11-27T21:36:19,337 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/85d4516e747c4232bf8555e1c80c1945 is 50, key is test_row_1/B:col10/1732743376883/Put/seqid=0 2024-11-27T21:36:19,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742274_1450 (size=9857) 2024-11-27T21:36:19,740 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/85d4516e747c4232bf8555e1c80c1945 2024-11-27T21:36:19,746 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/f1f68e2095c64963b6e7e36dc306f311 is 50, key is test_row_1/C:col10/1732743376883/Put/seqid=0 2024-11-27T21:36:19,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742275_1451 (size=9857) 2024-11-27T21:36:20,149 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/f1f68e2095c64963b6e7e36dc306f311 2024-11-27T21:36:20,153 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/A/54c8436e93274722a5d87489354dd043 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/54c8436e93274722a5d87489354dd043 2024-11-27T21:36:20,156 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/54c8436e93274722a5d87489354dd043, entries=100, sequenceid=342, filesize=22.0 K 2024-11-27T21:36:20,157 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/B/85d4516e747c4232bf8555e1c80c1945 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/85d4516e747c4232bf8555e1c80c1945 2024-11-27T21:36:20,159 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/85d4516e747c4232bf8555e1c80c1945, entries=100, sequenceid=342, filesize=9.6 K 2024-11-27T21:36:20,160 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/.tmp/C/f1f68e2095c64963b6e7e36dc306f311 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/f1f68e2095c64963b6e7e36dc306f311 2024-11-27T21:36:20,162 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/f1f68e2095c64963b6e7e36dc306f311, entries=100, sequenceid=342, filesize=9.6 K 2024-11-27T21:36:20,163 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for f9b9357dca19fe90b610172c1465ef7c in 1648ms, sequenceid=342, compaction requested=true 2024-11-27T21:36:20,164 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/21027248c3d74c79963bc7530b9ef8f5, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/0c26a6577f804813a2d786a100af8e7d, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/cffcaa3e5adf4679af73ee39e56feac8, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/9954eb11b40046679ba847a6d96504ab, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/01c0c77496764d46a126dd00f57db203, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/e9f2aed1d7d94d2884265b9ee63e79f5, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/fab7050e16d24d95b6ca0f41190b92a8, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/12aefea8bfd94e098e847ddc04a0224c, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/d6967770ad8d4a90af4cc0d47c8dc2fc, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/4a40798050c94b1ba7b98c12f95a2f5d, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/76a32d2d0e044604a719cf87a4d564af, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/be57d82bdc294942aba81affb5b84302, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/1dbe25d1aeef479d9ff1f119d3486c11, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/ea919ac1eaa840ad9881fa5405a5b018, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/d33616f433904988a4895f47b43ad6b9, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/c7332f0fb71c4965b60dbfe6823888b3, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/7d7350b36d184b6fb40491ee8f2b949c, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/0b68645d8b9d4b89b74e07b8bc92dd1a, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/aef9f288ef2641758ef8bc6b85177c25, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/0e5a1572df1f41169e81fc28f242b3e6, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/bc768e865b1b45efb4fbea02d214a033] to archive 2024-11-27T21:36:20,164 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T21:36:20,165 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/21027248c3d74c79963bc7530b9ef8f5 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/21027248c3d74c79963bc7530b9ef8f5 2024-11-27T21:36:20,166 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/0c26a6577f804813a2d786a100af8e7d to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/0c26a6577f804813a2d786a100af8e7d 2024-11-27T21:36:20,167 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/cffcaa3e5adf4679af73ee39e56feac8 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/cffcaa3e5adf4679af73ee39e56feac8 2024-11-27T21:36:20,168 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/9954eb11b40046679ba847a6d96504ab to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/9954eb11b40046679ba847a6d96504ab 2024-11-27T21:36:20,169 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/01c0c77496764d46a126dd00f57db203 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/01c0c77496764d46a126dd00f57db203 2024-11-27T21:36:20,169 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/e9f2aed1d7d94d2884265b9ee63e79f5 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/e9f2aed1d7d94d2884265b9ee63e79f5 2024-11-27T21:36:20,170 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/fab7050e16d24d95b6ca0f41190b92a8 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/fab7050e16d24d95b6ca0f41190b92a8 2024-11-27T21:36:20,171 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/12aefea8bfd94e098e847ddc04a0224c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/12aefea8bfd94e098e847ddc04a0224c 2024-11-27T21:36:20,172 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/d6967770ad8d4a90af4cc0d47c8dc2fc to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/d6967770ad8d4a90af4cc0d47c8dc2fc 2024-11-27T21:36:20,172 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/4a40798050c94b1ba7b98c12f95a2f5d to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/4a40798050c94b1ba7b98c12f95a2f5d 2024-11-27T21:36:20,173 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/76a32d2d0e044604a719cf87a4d564af to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/76a32d2d0e044604a719cf87a4d564af 2024-11-27T21:36:20,174 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/be57d82bdc294942aba81affb5b84302 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/be57d82bdc294942aba81affb5b84302 2024-11-27T21:36:20,175 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/1dbe25d1aeef479d9ff1f119d3486c11 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/1dbe25d1aeef479d9ff1f119d3486c11 2024-11-27T21:36:20,175 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/ea919ac1eaa840ad9881fa5405a5b018 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/ea919ac1eaa840ad9881fa5405a5b018 2024-11-27T21:36:20,176 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/d33616f433904988a4895f47b43ad6b9 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/d33616f433904988a4895f47b43ad6b9 2024-11-27T21:36:20,177 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/c7332f0fb71c4965b60dbfe6823888b3 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/c7332f0fb71c4965b60dbfe6823888b3 2024-11-27T21:36:20,178 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/7d7350b36d184b6fb40491ee8f2b949c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/7d7350b36d184b6fb40491ee8f2b949c 2024-11-27T21:36:20,179 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/0b68645d8b9d4b89b74e07b8bc92dd1a to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/0b68645d8b9d4b89b74e07b8bc92dd1a 2024-11-27T21:36:20,179 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/aef9f288ef2641758ef8bc6b85177c25 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/aef9f288ef2641758ef8bc6b85177c25 2024-11-27T21:36:20,180 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/0e5a1572df1f41169e81fc28f242b3e6 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/0e5a1572df1f41169e81fc28f242b3e6 2024-11-27T21:36:20,181 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/bc768e865b1b45efb4fbea02d214a033 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/bc768e865b1b45efb4fbea02d214a033 2024-11-27T21:36:20,182 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/aa4d407982d842fcbec4b0994ee9936a, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/21703d718246482a880638d3a7718c47, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/e283383e8a39481ba63bc1ea8b900cb5, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/44c573df0b984de68778e636f8e1ab81, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/c7f4756b6f764330bcf0a00cf5ac0a04, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/d7e30b2d41314c36a5fde82a1e805b2a, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/56cd2647591049e0a6c668cd86401a7b, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/f8891a4503864266aea1ab287e48f495, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/8d3d8c10b8db4c65863f364c02bfebac, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/3b998edb0ea046ee8f95a3619b54d312, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/c8fcfe69faee4666a3320a31410cbf62, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/13bd8897c45741d79707d3e4bc4eb1d9, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/3f83c44b6e4d4af190382807ee63d5e9, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/9844b9e7749c4b4e896da6a08a09163f, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/0606ad91817c4fc88ebdec25ec928d6e, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/e0a5f795f0e14a98a8dac55784a11356, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/350d5484e9c34a61b8c2838ac72a6b7c, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/31dd7b665c904118bc17d37f6a11a28d, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/a22fb9abd30c4c11b7ce844618e28c79, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/698b68b181ab4878a19d5a867d3864e7, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/049866951b174724b8d414c749f0063f] to archive 2024-11-27T21:36:20,183 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T21:36:20,184 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/aa4d407982d842fcbec4b0994ee9936a to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/aa4d407982d842fcbec4b0994ee9936a 2024-11-27T21:36:20,185 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/21703d718246482a880638d3a7718c47 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/21703d718246482a880638d3a7718c47 2024-11-27T21:36:20,185 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/e283383e8a39481ba63bc1ea8b900cb5 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/e283383e8a39481ba63bc1ea8b900cb5 2024-11-27T21:36:20,186 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/44c573df0b984de68778e636f8e1ab81 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/44c573df0b984de68778e636f8e1ab81 2024-11-27T21:36:20,187 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/c7f4756b6f764330bcf0a00cf5ac0a04 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/c7f4756b6f764330bcf0a00cf5ac0a04 2024-11-27T21:36:20,188 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/d7e30b2d41314c36a5fde82a1e805b2a to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/d7e30b2d41314c36a5fde82a1e805b2a 2024-11-27T21:36:20,189 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/56cd2647591049e0a6c668cd86401a7b to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/56cd2647591049e0a6c668cd86401a7b 2024-11-27T21:36:20,189 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/f8891a4503864266aea1ab287e48f495 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/f8891a4503864266aea1ab287e48f495 2024-11-27T21:36:20,190 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/8d3d8c10b8db4c65863f364c02bfebac to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/8d3d8c10b8db4c65863f364c02bfebac 2024-11-27T21:36:20,191 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/3b998edb0ea046ee8f95a3619b54d312 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/3b998edb0ea046ee8f95a3619b54d312 2024-11-27T21:36:20,192 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/c8fcfe69faee4666a3320a31410cbf62 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/c8fcfe69faee4666a3320a31410cbf62 2024-11-27T21:36:20,192 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/13bd8897c45741d79707d3e4bc4eb1d9 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/13bd8897c45741d79707d3e4bc4eb1d9 2024-11-27T21:36:20,193 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/3f83c44b6e4d4af190382807ee63d5e9 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/3f83c44b6e4d4af190382807ee63d5e9 2024-11-27T21:36:20,194 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/9844b9e7749c4b4e896da6a08a09163f to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/9844b9e7749c4b4e896da6a08a09163f 2024-11-27T21:36:20,195 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/0606ad91817c4fc88ebdec25ec928d6e to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/0606ad91817c4fc88ebdec25ec928d6e 2024-11-27T21:36:20,195 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/e0a5f795f0e14a98a8dac55784a11356 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/e0a5f795f0e14a98a8dac55784a11356 2024-11-27T21:36:20,196 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/350d5484e9c34a61b8c2838ac72a6b7c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/350d5484e9c34a61b8c2838ac72a6b7c 2024-11-27T21:36:20,197 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/31dd7b665c904118bc17d37f6a11a28d to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/31dd7b665c904118bc17d37f6a11a28d 2024-11-27T21:36:20,198 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/a22fb9abd30c4c11b7ce844618e28c79 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/a22fb9abd30c4c11b7ce844618e28c79 2024-11-27T21:36:20,198 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/698b68b181ab4878a19d5a867d3864e7 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/698b68b181ab4878a19d5a867d3864e7 2024-11-27T21:36:20,199 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/049866951b174724b8d414c749f0063f to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/049866951b174724b8d414c749f0063f 2024-11-27T21:36:20,200 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/8bc59e85e42644599830c06e6a62d6e4, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/1e6432ce99864782af8c183a910a8071, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/bc050f7cbf2e4c83867bd8acad07e40d, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/061e32d4666c4b00ba58102dcca54e15, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/0d14d18731b0430facbaf7e0d6525c8e, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/2ee152e6a5a94e789ab6871afbb13542, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/1b71d38516e844c09a4a6a5cf7aaf22a, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/788289743849467fb62fe7c2f9f8d197, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/90bcbc96ce3f4c2ba035e6d3f22f4f02, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/a411f23c3a0c4c9694eb1f70c95d822c, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/cd61ac3a34524d13b3140a41d589a8e1, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/1f19438ae4d54c7091ad1a0c1b910892, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/e614ff4d32ba44bca26aed5ccd574c74, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/9f61529dfd16420a8f5b2a7bc7818b48, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/5daa316bceb249b6a37d29f116bf9872, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/c24c6ff93e314371a8e58530eb361d43, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/a02ddcad3e8643d490253832ed54d1ed, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/a100b3bd77904373b17eb8bbca026900, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/6e5648c28fd94297a0c3d4f14c9f463f, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/a2fa3fc4786f4c0e9bf2798221617ea9, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/36598795ee5b40c9a8c252d1cb3b2d0a] to archive 2024-11-27T21:36:20,201 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T21:36:20,202 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/8bc59e85e42644599830c06e6a62d6e4 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/8bc59e85e42644599830c06e6a62d6e4 2024-11-27T21:36:20,202 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/1e6432ce99864782af8c183a910a8071 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/1e6432ce99864782af8c183a910a8071 2024-11-27T21:36:20,203 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/bc050f7cbf2e4c83867bd8acad07e40d to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/bc050f7cbf2e4c83867bd8acad07e40d 2024-11-27T21:36:20,204 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/061e32d4666c4b00ba58102dcca54e15 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/061e32d4666c4b00ba58102dcca54e15 2024-11-27T21:36:20,204 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/0d14d18731b0430facbaf7e0d6525c8e to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/0d14d18731b0430facbaf7e0d6525c8e 2024-11-27T21:36:20,205 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/2ee152e6a5a94e789ab6871afbb13542 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/2ee152e6a5a94e789ab6871afbb13542 2024-11-27T21:36:20,206 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/1b71d38516e844c09a4a6a5cf7aaf22a to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/1b71d38516e844c09a4a6a5cf7aaf22a 2024-11-27T21:36:20,206 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/788289743849467fb62fe7c2f9f8d197 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/788289743849467fb62fe7c2f9f8d197 2024-11-27T21:36:20,207 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/90bcbc96ce3f4c2ba035e6d3f22f4f02 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/90bcbc96ce3f4c2ba035e6d3f22f4f02 2024-11-27T21:36:20,208 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/a411f23c3a0c4c9694eb1f70c95d822c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/a411f23c3a0c4c9694eb1f70c95d822c 2024-11-27T21:36:20,209 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/cd61ac3a34524d13b3140a41d589a8e1 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/cd61ac3a34524d13b3140a41d589a8e1 2024-11-27T21:36:20,209 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/1f19438ae4d54c7091ad1a0c1b910892 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/1f19438ae4d54c7091ad1a0c1b910892 2024-11-27T21:36:20,210 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/e614ff4d32ba44bca26aed5ccd574c74 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/e614ff4d32ba44bca26aed5ccd574c74 2024-11-27T21:36:20,211 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/9f61529dfd16420a8f5b2a7bc7818b48 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/9f61529dfd16420a8f5b2a7bc7818b48 2024-11-27T21:36:20,212 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/5daa316bceb249b6a37d29f116bf9872 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/5daa316bceb249b6a37d29f116bf9872 2024-11-27T21:36:20,212 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/c24c6ff93e314371a8e58530eb361d43 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/c24c6ff93e314371a8e58530eb361d43 2024-11-27T21:36:20,213 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/a02ddcad3e8643d490253832ed54d1ed to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/a02ddcad3e8643d490253832ed54d1ed 2024-11-27T21:36:20,214 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/a100b3bd77904373b17eb8bbca026900 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/a100b3bd77904373b17eb8bbca026900 2024-11-27T21:36:20,215 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/6e5648c28fd94297a0c3d4f14c9f463f to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/6e5648c28fd94297a0c3d4f14c9f463f 2024-11-27T21:36:20,215 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/a2fa3fc4786f4c0e9bf2798221617ea9 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/a2fa3fc4786f4c0e9bf2798221617ea9 2024-11-27T21:36:20,216 DEBUG [StoreCloser-TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/36598795ee5b40c9a8c252d1cb3b2d0a to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/36598795ee5b40c9a8c252d1cb3b2d0a 2024-11-27T21:36:20,220 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/recovered.edits/345.seqid, newMaxSeqId=345, maxSeqId=4 2024-11-27T21:36:20,220 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. 2024-11-27T21:36:20,220 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1676): Region close journal for f9b9357dca19fe90b610172c1465ef7c: Waiting for close lock at 1732743377055Running coprocessor pre-close hooks at 1732743377055Disabling compacts and flushes for region at 1732743377055Disabling writes for close at 1732743378515 (+1460 ms)Obtaining lock to block concurrent updates at 1732743378515Preparing flush snapshotting stores in f9b9357dca19fe90b610172c1465ef7c at 1732743378515Finished memstore snapshotting TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1732743378515Flushing stores of TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c. at 1732743378516 (+1 ms)Flushing f9b9357dca19fe90b610172c1465ef7c/A: creating writer at 1732743378516Flushing mob file f9b9357dca19fe90b610172c1465ef7c/A: appending metadata at 1732743378519 (+3 ms)Flushing mob file f9b9357dca19fe90b610172c1465ef7c/A: closing flushed file at 1732743378519Flushing f9b9357dca19fe90b610172c1465ef7c/A: appending metadata at 1732743378927 (+408 ms)Flushing f9b9357dca19fe90b610172c1465ef7c/A: closing flushed file at 1732743378927Flushing f9b9357dca19fe90b610172c1465ef7c/B: creating writer at 1732743379334 (+407 ms)Flushing f9b9357dca19fe90b610172c1465ef7c/B: appending metadata at 1732743379336 (+2 ms)Flushing f9b9357dca19fe90b610172c1465ef7c/B: closing flushed file at 1732743379336Flushing f9b9357dca19fe90b610172c1465ef7c/C: creating writer at 1732743379743 (+407 ms)Flushing f9b9357dca19fe90b610172c1465ef7c/C: appending metadata at 1732743379745 (+2 ms)Flushing f9b9357dca19fe90b610172c1465ef7c/C: closing flushed file at 1732743379745Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ed282e9: reopening flushed file at 1732743380152 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6fbb9131: reopening flushed file at 1732743380156 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@358a945e: reopening flushed file at 1732743380159 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for f9b9357dca19fe90b610172c1465ef7c in 1648ms, sequenceid=342, compaction requested=true at 1732743380163 (+4 ms)Writing region close event to WAL at 1732743380217 (+54 ms)Running coprocessor post-close hooks at 1732743380220 (+3 ms)Closed at 1732743380220 2024-11-27T21:36:20,222 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(157): Closed f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:36:20,222 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=f9b9357dca19fe90b610172c1465ef7c, regionState=CLOSED 2024-11-27T21:36:20,223 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=122, ppid=121, state=RUNNABLE, hasLock=false; CloseRegionProcedure f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 because future has completed 2024-11-27T21:36:20,225 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-27T21:36:20,225 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; CloseRegionProcedure f9b9357dca19fe90b610172c1465ef7c, server=4585c39bed03,39097,1732743267048 in 3.3210 sec 2024-11-27T21:36:20,227 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=121, resume processing ppid=120 2024-11-27T21:36:20,227 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, ppid=120, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f9b9357dca19fe90b610172c1465ef7c, UNASSIGN in 3.3240 sec 2024-11-27T21:36:20,228 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-27T21:36:20,228 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 3.3270 sec 2024-11-27T21:36:20,229 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732743380229"}]},"ts":"1732743380229"} 2024-11-27T21:36:20,230 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-27T21:36:20,230 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-27T21:36:20,232 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 3.3360 sec 2024-11-27T21:36:21,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-27T21:36:21,053 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:36:21,054 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-27T21:36:21,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:36:21,055 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=123, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:36:21,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-27T21:36:21,056 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=123, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:36:21,058 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:36:21,060 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A, FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B, FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C, FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/recovered.edits] 2024-11-27T21:36:21,062 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/0eb5456ddcec4f87a891156f90fd0b69 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/0eb5456ddcec4f87a891156f90fd0b69 2024-11-27T21:36:21,063 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/4e7145f5ef04430b858cffe8de9edb04 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/4e7145f5ef04430b858cffe8de9edb04 2024-11-27T21:36:21,063 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/54c8436e93274722a5d87489354dd043 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/54c8436e93274722a5d87489354dd043 2024-11-27T21:36:21,064 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/d4443b02c62c43229c61a97f051a763c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/A/d4443b02c62c43229c61a97f051a763c 2024-11-27T21:36:21,066 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/85d4516e747c4232bf8555e1c80c1945 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/85d4516e747c4232bf8555e1c80c1945 2024-11-27T21:36:21,067 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/9d66757bd7fa4640979b06caa7cbd369 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/9d66757bd7fa4640979b06caa7cbd369 2024-11-27T21:36:21,067 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/bd97371522cd481e9a4b5605b27a470e to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/bd97371522cd481e9a4b5605b27a470e 2024-11-27T21:36:21,068 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/e8f64b1f196d48e984f5e5f662ac0902 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/B/e8f64b1f196d48e984f5e5f662ac0902 2024-11-27T21:36:21,070 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/4069d514a65c4a4e9f3b65b0745ff7ba to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/4069d514a65c4a4e9f3b65b0745ff7ba 2024-11-27T21:36:21,071 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/c3921e054d1c49d8884dc68792abde80 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/c3921e054d1c49d8884dc68792abde80 2024-11-27T21:36:21,072 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/eeb46a486e304d5e848e0f9c74550a37 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/eeb46a486e304d5e848e0f9c74550a37 2024-11-27T21:36:21,073 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/f1f68e2095c64963b6e7e36dc306f311 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/C/f1f68e2095c64963b6e7e36dc306f311 2024-11-27T21:36:21,074 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/recovered.edits/345.seqid to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c/recovered.edits/345.seqid 2024-11-27T21:36:21,075 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:36:21,075 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-27T21:36:21,075 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-27T21:36:21,076 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-27T21:36:21,078 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112709982fe646f64c14837fbf754b883958_f9b9357dca19fe90b610172c1465ef7c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112709982fe646f64c14837fbf754b883958_f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:36:21,078 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411272a67830b56fa4b62b84101bc706659cd_f9b9357dca19fe90b610172c1465ef7c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411272a67830b56fa4b62b84101bc706659cd_f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:36:21,079 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411274029b6863c51432098449af8097c4dc5_f9b9357dca19fe90b610172c1465ef7c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411274029b6863c51432098449af8097c4dc5_f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:36:21,080 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112748640639c91b4fa68a740821c0aaed0f_f9b9357dca19fe90b610172c1465ef7c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112748640639c91b4fa68a740821c0aaed0f_f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:36:21,081 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411274b9ea8f6e0fe4f4ab7ea70ad9916e426_f9b9357dca19fe90b610172c1465ef7c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411274b9ea8f6e0fe4f4ab7ea70ad9916e426_f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:36:21,081 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411275d7564d5169e44589334a0e5338c3ca9_f9b9357dca19fe90b610172c1465ef7c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411275d7564d5169e44589334a0e5338c3ca9_f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:36:21,082 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411278f26c69e7f624d45bda13b3e0f3290c6_f9b9357dca19fe90b610172c1465ef7c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411278f26c69e7f624d45bda13b3e0f3290c6_f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:36:21,083 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279bde3d40c3b343fea5cdd91052c12cb7_f9b9357dca19fe90b610172c1465ef7c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279bde3d40c3b343fea5cdd91052c12cb7_f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:36:21,084 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a10bf3b484604b6da8401d44f870b0f9_f9b9357dca19fe90b610172c1465ef7c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a10bf3b484604b6da8401d44f870b0f9_f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:36:21,085 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d0b5e56b9cec4fff8fc2212dd8ba6606_f9b9357dca19fe90b610172c1465ef7c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d0b5e56b9cec4fff8fc2212dd8ba6606_f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:36:21,085 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d3785cec459245329abe66d68a8302f8_f9b9357dca19fe90b610172c1465ef7c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d3785cec459245329abe66d68a8302f8_f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:36:21,086 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d3edbdeea7b545f384116330929fd158_f9b9357dca19fe90b610172c1465ef7c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d3edbdeea7b545f384116330929fd158_f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:36:21,087 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127e43ad6d641184304b3abc2adb76da562_f9b9357dca19fe90b610172c1465ef7c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127e43ad6d641184304b3abc2adb76da562_f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:36:21,088 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f3b5cb37a1684db2a6365da0c6ae0994_f9b9357dca19fe90b610172c1465ef7c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f3b5cb37a1684db2a6365da0c6ae0994_f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:36:21,089 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f6d84f852c674785a6c1ab47f09f7c7b_f9b9357dca19fe90b610172c1465ef7c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f6d84f852c674785a6c1ab47f09f7c7b_f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:36:21,090 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f8866f3cc05d4e8d8681cc0c52fc15e5_f9b9357dca19fe90b610172c1465ef7c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f8866f3cc05d4e8d8681cc0c52fc15e5_f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:36:21,090 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127fa2b8d83e4bb45d6af05f18b6dad7a45_f9b9357dca19fe90b610172c1465ef7c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127fa2b8d83e4bb45d6af05f18b6dad7a45_f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:36:21,091 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127fe336c09412042af9a326821d1379087_f9b9357dca19fe90b610172c1465ef7c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127fe336c09412042af9a326821d1379087_f9b9357dca19fe90b610172c1465ef7c 2024-11-27T21:36:21,091 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-27T21:36:21,093 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=123, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:36:21,094 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-27T21:36:21,096 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-27T21:36:21,097 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=123, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:36:21,097 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-27T21:36:21,097 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732743381097"}]},"ts":"9223372036854775807"} 2024-11-27T21:36:21,098 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-27T21:36:21,098 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f9b9357dca19fe90b610172c1465ef7c, NAME => 'TestAcidGuarantees,,1732743351672.f9b9357dca19fe90b610172c1465ef7c.', STARTKEY => '', ENDKEY => ''}] 2024-11-27T21:36:21,098 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-27T21:36:21,098 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732743381098"}]},"ts":"9223372036854775807"} 2024-11-27T21:36:21,100 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-27T21:36:21,100 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=123, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:36:21,101 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 46 msec 2024-11-27T21:36:21,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-27T21:36:21,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-27T21:36:21,163 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:36:21,172 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=240 (was 240), OpenFileDescriptor=453 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=308 (was 332), ProcessCount=11 (was 11), AvailableMemoryMB=8248 (was 8294) 2024-11-27T21:36:21,180 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=240, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=308, ProcessCount=11, AvailableMemoryMB=8248 2024-11-27T21:36:21,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T21:36:21,181 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T21:36:21,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=124, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-27T21:36:21,183 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T21:36:21,183 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:36:21,183 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 124 2024-11-27T21:36:21,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-27T21:36:21,183 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T21:36:21,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742276_1452 (size=959) 2024-11-27T21:36:21,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-27T21:36:21,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-27T21:36:21,590 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d 2024-11-27T21:36:21,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742277_1453 (size=53) 2024-11-27T21:36:21,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-27T21:36:21,995 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T21:36:21,995 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing b39500f7a69456fd382cec5091ca715b, disabling compactions & flushes 2024-11-27T21:36:21,995 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:21,995 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:21,995 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. after waiting 0 ms 2024-11-27T21:36:21,995 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:21,995 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:21,995 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for b39500f7a69456fd382cec5091ca715b: Waiting for close lock at 1732743381995Disabling compacts and flushes for region at 1732743381995Disabling writes for close at 1732743381995Writing region close event to WAL at 1732743381995Closed at 1732743381995 2024-11-27T21:36:21,996 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T21:36:21,996 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732743381996"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732743381996"}]},"ts":"1732743381996"} 2024-11-27T21:36:21,997 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T21:36:21,998 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T21:36:21,998 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732743381998"}]},"ts":"1732743381998"} 2024-11-27T21:36:21,999 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-27T21:36:21,999 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b39500f7a69456fd382cec5091ca715b, ASSIGN}] 2024-11-27T21:36:22,000 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b39500f7a69456fd382cec5091ca715b, ASSIGN 2024-11-27T21:36:22,001 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b39500f7a69456fd382cec5091ca715b, ASSIGN; state=OFFLINE, location=4585c39bed03,39097,1732743267048; forceNewPlan=false, retain=false 2024-11-27T21:36:22,151 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=b39500f7a69456fd382cec5091ca715b, regionState=OPENING, regionLocation=4585c39bed03,39097,1732743267048 2024-11-27T21:36:22,153 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b39500f7a69456fd382cec5091ca715b, ASSIGN because future has completed 2024-11-27T21:36:22,153 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; OpenRegionProcedure b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048}] 2024-11-27T21:36:22,307 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:22,307 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7752): Opening region: {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} 2024-11-27T21:36:22,308 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b39500f7a69456fd382cec5091ca715b 2024-11-27T21:36:22,308 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T21:36:22,308 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7794): checking encryption for b39500f7a69456fd382cec5091ca715b 2024-11-27T21:36:22,308 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7797): checking classloading for b39500f7a69456fd382cec5091ca715b 2024-11-27T21:36:22,309 INFO [StoreOpener-b39500f7a69456fd382cec5091ca715b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b39500f7a69456fd382cec5091ca715b 2024-11-27T21:36:22,310 INFO [StoreOpener-b39500f7a69456fd382cec5091ca715b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b39500f7a69456fd382cec5091ca715b columnFamilyName A 2024-11-27T21:36:22,310 DEBUG [StoreOpener-b39500f7a69456fd382cec5091ca715b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:36:22,310 INFO [StoreOpener-b39500f7a69456fd382cec5091ca715b-1 {}] regionserver.HStore(327): Store=b39500f7a69456fd382cec5091ca715b/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:36:22,310 INFO [StoreOpener-b39500f7a69456fd382cec5091ca715b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b39500f7a69456fd382cec5091ca715b 2024-11-27T21:36:22,311 INFO [StoreOpener-b39500f7a69456fd382cec5091ca715b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b39500f7a69456fd382cec5091ca715b columnFamilyName B 2024-11-27T21:36:22,311 DEBUG [StoreOpener-b39500f7a69456fd382cec5091ca715b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:36:22,312 INFO [StoreOpener-b39500f7a69456fd382cec5091ca715b-1 {}] regionserver.HStore(327): Store=b39500f7a69456fd382cec5091ca715b/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:36:22,312 INFO [StoreOpener-b39500f7a69456fd382cec5091ca715b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b39500f7a69456fd382cec5091ca715b 2024-11-27T21:36:22,312 INFO [StoreOpener-b39500f7a69456fd382cec5091ca715b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b39500f7a69456fd382cec5091ca715b columnFamilyName C 2024-11-27T21:36:22,312 DEBUG [StoreOpener-b39500f7a69456fd382cec5091ca715b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:36:22,313 INFO [StoreOpener-b39500f7a69456fd382cec5091ca715b-1 {}] regionserver.HStore(327): Store=b39500f7a69456fd382cec5091ca715b/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:36:22,313 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1038): replaying wal for b39500f7a69456fd382cec5091ca715b 2024-11-27T21:36:22,313 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b 2024-11-27T21:36:22,313 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b 2024-11-27T21:36:22,314 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1048): stopping wal replay for b39500f7a69456fd382cec5091ca715b 2024-11-27T21:36:22,314 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1060): Cleaning up temporary data for b39500f7a69456fd382cec5091ca715b 2024-11-27T21:36:22,314 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T21:36:22,315 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1093): writing seq id for b39500f7a69456fd382cec5091ca715b 2024-11-27T21:36:22,317 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T21:36:22,317 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1114): Opened b39500f7a69456fd382cec5091ca715b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59652220, jitterRate=-0.11111265420913696}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T21:36:22,317 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b39500f7a69456fd382cec5091ca715b 2024-11-27T21:36:22,318 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1006): Region open journal for b39500f7a69456fd382cec5091ca715b: Running coprocessor pre-open hook at 1732743382308Writing region info on filesystem at 1732743382308Initializing all the Stores at 1732743382308Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732743382309 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732743382309Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732743382309Cleaning up temporary data from old regions at 1732743382314 (+5 ms)Running coprocessor post-open hooks at 1732743382317 (+3 ms)Region opened successfully at 1732743382318 (+1 ms) 2024-11-27T21:36:22,318 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., pid=126, masterSystemTime=1732743382304 2024-11-27T21:36:22,320 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:22,320 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:22,320 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=b39500f7a69456fd382cec5091ca715b, regionState=OPEN, openSeqNum=2, regionLocation=4585c39bed03,39097,1732743267048 2024-11-27T21:36:22,322 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=126, ppid=125, state=RUNNABLE, hasLock=false; OpenRegionProcedure b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 because future has completed 2024-11-27T21:36:22,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-27T21:36:22,323 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-27T21:36:22,323 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; OpenRegionProcedure b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 in 169 msec 2024-11-27T21:36:22,325 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=125, resume processing ppid=124 2024-11-27T21:36:22,325 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, ppid=124, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b39500f7a69456fd382cec5091ca715b, ASSIGN in 324 msec 2024-11-27T21:36:22,325 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T21:36:22,326 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732743382326"}]},"ts":"1732743382326"} 2024-11-27T21:36:22,327 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-27T21:36:22,327 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T21:36:22,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1460 sec 2024-11-27T21:36:23,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-27T21:36:23,343 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:36:23,344 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@277faa65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:23,344 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:36:23,344 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:36:23,345 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:36:23,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:36:23,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:36:23,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@628e6b21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:23,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:36:23,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:36:23,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:23,347 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41210, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:36:23,347 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72a7afdd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:23,347 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:36:23,348 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:36:23,348 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:36:23,349 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39124, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:36:23,349 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T21:36:23,350 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 4585c39bed03,34433,1732743266181 2024-11-27T21:36:23,350 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@48ce26f9 2024-11-27T21:36:23,350 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T21:36:23,351 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41216, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T21:36:23,352 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@269ecf23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:23,352 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:36:23,352 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:36:23,352 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:36:23,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:36:23,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:36:23,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18b39bad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:23,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:36:23,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:36:23,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:23,354 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41240, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:36:23,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61de62a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:23,354 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@695dbdde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:23,355 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:36:23,355 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:36:23,355 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:36:23,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:36:23,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:36:23,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3095b659, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:23,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:36:23,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:36:23,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:23,356 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41264, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:36:23,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f980cd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:23,357 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594674bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:23,357 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:36:23,357 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:36:23,358 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:36:23,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:36:23,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:36:23,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@254cad95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:23,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:36:23,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:36:23,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:23,359 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41284, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:36:23,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29725563, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:23,360 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@119ad987, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:23,360 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:36:23,360 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:36:23,361 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:36:23,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:36:23,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:36:23,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8f820ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:23,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:36:23,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:36:23,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:23,361 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41298, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:36:23,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c8ade5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:23,362 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26c4ef34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:23,362 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:36:23,363 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:36:23,363 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:36:23,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:36:23,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:36:23,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@608ca0d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:23,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:36:23,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:36:23,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:23,364 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41322, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:36:23,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a756ec6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:23,365 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38404fa5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:23,365 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:36:23,365 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:36:23,366 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:36:23,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:36:23,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:36:23,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e48c01d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:23,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:36:23,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:36:23,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:23,367 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41348, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:36:23,367 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@208bf58d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:23,368 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27ae0787, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:23,368 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:36:23,368 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:36:23,368 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:36:23,368 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:36:23,368 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:36:23,369 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab15048, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:23,369 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:36:23,369 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:36:23,369 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:23,369 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41370, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:36:23,370 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48680116, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:23,370 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1627c26b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:23,370 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:36:23,370 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:36:23,371 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:36:23,371 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:36:23,371 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:36:23,371 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f8084ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:23,371 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:36:23,371 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:36:23,371 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:23,372 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41386, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:36:23,372 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dae8a57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:23,373 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5993889b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:23,373 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:36:23,373 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:36:23,374 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:36:23,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:36:23,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:36:23,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7baa2670, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:23,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:36:23,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:36:23,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:23,375 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41394, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:36:23,375 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e2e98fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:23,376 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@117399b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:23,376 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:36:23,376 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:36:23,376 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:36:23,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:36:23,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:36:23,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bb7fbf0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:23,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:36:23,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:36:23,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:23,377 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41426, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:36:23,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29be8b14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:23,378 DEBUG [Thread-2037 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:36:23,378 DEBUG [Thread-2039 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:36:23,378 DEBUG [Thread-2028 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:36:23,383 DEBUG [Thread-2030 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:36:23,383 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:36:23,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:36:23,383 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:36:23,383 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:36:23,383 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:36:23,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:36:23,383 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:36:23,383 DEBUG [Thread-2041 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:36:23,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:36:23,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:36:23,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-27T21:36:23,384 DEBUG [Thread-2043 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:36:23,384 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39130, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:36:23,384 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39156, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:36:23,384 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39142, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:36:23,385 DEBUG [Thread-2024 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:36:23,385 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:36:23,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-27T21:36:23,385 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39168, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:36:23,385 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:36:23,385 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:36:23,385 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:36:23,385 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:36:23,386 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39184, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:36:23,386 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39200, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:36:23,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2] 2024-11-27T21:36:23,386 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:36:23,386 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:36:23,386 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:36:23,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2] 2024-11-27T21:36:23,386 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:36:23,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2] 2024-11-27T21:36:23,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2] 2024-11-27T21:36:23,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2] 2024-11-27T21:36:23,389 DEBUG [Thread-2032 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:36:23,389 DEBUG [Thread-2026 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:36:23,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2] 2024-11-27T21:36:23,390 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39204, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:36:23,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:36:23,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:36:23,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:36:23,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:36:23,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2] 2024-11-27T21:36:23,393 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39208, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:36:23,393 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39212, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:36:23,394 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2] 2024-11-27T21:36:23,395 DEBUG [Thread-2035 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:36:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:36:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:36:23,396 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39216, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:36:23,398 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2] 2024-11-27T21:36:23,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on b39500f7a69456fd382cec5091ca715b 2024-11-27T21:36:23,399 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:36:23,399 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b39500f7a69456fd382cec5091ca715b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T21:36:23,403 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2] 2024-11-27T21:36:23,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/e2cb3d5f3fa14aba965026037e408ebb is 50, key is test_row_0/A:col10/1732743383395/Put/seqid=0 2024-11-27T21:36:23,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742278_1454 (size=14337) 2024-11-27T21:36:23,428 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/e2cb3d5f3fa14aba965026037e408ebb 2024-11-27T21:36:23,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:23,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743443428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:23,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:23,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:23,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:23,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:23,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743443428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:23,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:23,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:23,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:23,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743443429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:23,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:23,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:23,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:23,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:23,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:23,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1732743443430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:23,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:23,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:23,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:23,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:23,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743443434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:23,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:23,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:23,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:23,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/317838de3d024c6fb7b347da2c3899e4 is 50, key is test_row_0/B:col10/1732743383395/Put/seqid=0 2024-11-27T21:36:23,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742279_1455 (size=9657) 2024-11-27T21:36:23,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-27T21:36:23,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-27T21:36:23,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:23,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:23,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:23,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:23,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:23,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:23,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:23,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743443543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:23,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:23,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:23,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:23,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743443543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:23,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:23,549 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:23,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:23,549 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:23,549 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:23,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743443543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:23,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:23,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:23,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:23,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1732743443543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:23,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:23,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:23,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:23,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:23,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:23,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743443553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:23,558 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:23,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:23,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:23,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-27T21:36:23,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:23,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:23,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:23,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:23,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:23,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:23,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-27T21:36:23,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:23,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743443753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:23,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:23,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:23,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:23,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743443753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:23,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:23,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:23,755 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:23,755 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:23,755 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:23,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743443753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:23,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:23,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:23,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:23,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:23,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1732743443753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:23,756 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:23,756 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:23,756 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:23,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:23,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743443764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:23,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:23,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:23,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:23,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-27T21:36:23,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:23,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:23,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:23,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:23,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:23,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:23,858 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/317838de3d024c6fb7b347da2c3899e4 2024-11-27T21:36:23,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/3ccd53ef3fe140d699a6a39b94d0be2c is 50, key is test_row_0/C:col10/1732743383395/Put/seqid=0 2024-11-27T21:36:23,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742280_1456 (size=9657) 2024-11-27T21:36:23,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-27T21:36:23,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:23,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:23,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:23,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:23,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:23,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:24,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-27T21:36:24,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:24,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743444063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:24,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:24,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:24,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:24,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743444063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:24,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:24,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:24,070 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:24,070 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:24,070 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:24,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743444064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:24,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:24,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:24,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1732743444064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:24,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:24,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:24,070 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:24,070 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:24,070 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:24,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:24,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743444073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:24,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:24,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:24,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:24,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-27T21:36:24,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:24,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:24,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:24,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:24,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:24,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:24,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/3ccd53ef3fe140d699a6a39b94d0be2c 2024-11-27T21:36:24,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/e2cb3d5f3fa14aba965026037e408ebb as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/e2cb3d5f3fa14aba965026037e408ebb 2024-11-27T21:36:24,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/e2cb3d5f3fa14aba965026037e408ebb, entries=200, sequenceid=13, filesize=14.0 K 2024-11-27T21:36:24,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/317838de3d024c6fb7b347da2c3899e4 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/317838de3d024c6fb7b347da2c3899e4 2024-11-27T21:36:24,298 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/317838de3d024c6fb7b347da2c3899e4, entries=100, sequenceid=13, filesize=9.4 K 2024-11-27T21:36:24,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/3ccd53ef3fe140d699a6a39b94d0be2c as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/3ccd53ef3fe140d699a6a39b94d0be2c 2024-11-27T21:36:24,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/3ccd53ef3fe140d699a6a39b94d0be2c, entries=100, sequenceid=13, filesize=9.4 K 2024-11-27T21:36:24,303 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for b39500f7a69456fd382cec5091ca715b in 904ms, sequenceid=13, compaction requested=false 2024-11-27T21:36:24,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:24,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-27T21:36:24,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:24,304 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing b39500f7a69456fd382cec5091ca715b 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-27T21:36:24,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/98468e9e0ea0413187b3afc43e788dfe is 50, key is test_row_0/A:col10/1732743383433/Put/seqid=0 2024-11-27T21:36:24,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742281_1457 (size=12001) 2024-11-27T21:36:24,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-27T21:36:24,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on b39500f7a69456fd382cec5091ca715b 2024-11-27T21:36:24,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:24,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:24,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1732743444589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:24,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:24,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:24,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743444590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:24,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:24,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:24,602 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:24,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:24,602 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:24,602 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:24,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743444591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:24,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:24,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743444592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:24,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:24,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:24,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:24,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:24,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743444601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:24,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:24,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:24,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:24,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:24,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:24,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:24,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:24,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1732743444703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:24,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:24,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:24,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:24,712 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/98468e9e0ea0413187b3afc43e788dfe 2024-11-27T21:36:24,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/6f89273aa73d4120bf349574fdb8324a is 50, key is test_row_0/B:col10/1732743383433/Put/seqid=0 2024-11-27T21:36:24,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:24,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743444713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:24,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:24,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:24,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743444713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:24,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:24,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:24,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:24,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:24,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:24,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:24,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743444714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:24,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:24,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:24,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:24,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:24,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743444714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:24,723 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:24,723 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:24,723 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:24,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742282_1458 (size=12001) 2024-11-27T21:36:24,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:24,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1732743444913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:24,921 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:24,921 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:24,921 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:24,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:24,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743444933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:24,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:24,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:24,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:24,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743444933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:24,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:24,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:24,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:24,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:24,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743444933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:24,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:24,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:24,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:24,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:24,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:24,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743444933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:24,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:24,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:24,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:25,123 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/6f89273aa73d4120bf349574fdb8324a 2024-11-27T21:36:25,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/75824f2e34f94ddd860ef72b3a526371 is 50, key is test_row_0/C:col10/1732743383433/Put/seqid=0 2024-11-27T21:36:25,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742283_1459 (size=12001) 2024-11-27T21:36:25,212 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T21:36:25,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:25,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1732743445224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:25,229 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:25,229 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:25,229 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:25,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:25,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743445253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:25,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:25,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743445253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:25,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:25,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:25,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:25,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:25,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:25,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743445253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:25,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:25,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:25,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:25,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:25,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743445253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:25,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:25,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:25,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:25,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:25,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:25,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-27T21:36:25,542 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/75824f2e34f94ddd860ef72b3a526371 2024-11-27T21:36:25,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/98468e9e0ea0413187b3afc43e788dfe as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/98468e9e0ea0413187b3afc43e788dfe 2024-11-27T21:36:25,550 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/98468e9e0ea0413187b3afc43e788dfe, entries=150, sequenceid=37, filesize=11.7 K 2024-11-27T21:36:25,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/6f89273aa73d4120bf349574fdb8324a as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/6f89273aa73d4120bf349574fdb8324a 2024-11-27T21:36:25,555 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/6f89273aa73d4120bf349574fdb8324a, entries=150, sequenceid=37, filesize=11.7 K 2024-11-27T21:36:25,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/75824f2e34f94ddd860ef72b3a526371 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/75824f2e34f94ddd860ef72b3a526371 2024-11-27T21:36:25,559 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/75824f2e34f94ddd860ef72b3a526371, entries=150, sequenceid=37, filesize=11.7 K 2024-11-27T21:36:25,560 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for b39500f7a69456fd382cec5091ca715b in 1256ms, sequenceid=37, compaction requested=false 2024-11-27T21:36:25,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:25,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:25,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-27T21:36:25,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-27T21:36:25,563 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-27T21:36:25,563 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1750 sec 2024-11-27T21:36:25,565 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.1810 sec 2024-11-27T21:36:25,736 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T21:36:25,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on b39500f7a69456fd382cec5091ca715b 2024-11-27T21:36:25,737 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:36:25,737 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b39500f7a69456fd382cec5091ca715b 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-27T21:36:25,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/2919a52cf0774516acd56aba31cbbf9f is 50, key is test_row_0/A:col10/1732743384599/Put/seqid=0 2024-11-27T21:36:25,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742284_1460 (size=12001) 2024-11-27T21:36:25,755 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/2919a52cf0774516acd56aba31cbbf9f 2024-11-27T21:36:25,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/21d92481ae3c454e824c0ac32965be98 is 50, key is test_row_0/B:col10/1732743384599/Put/seqid=0 2024-11-27T21:36:25,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742285_1461 (size=12001) 2024-11-27T21:36:25,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:25,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743445800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:25,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:25,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743445801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:25,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:25,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743445801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:25,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:25,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:25,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:25,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:25,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:25,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:25,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:25,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:25,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:25,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:25,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1732743445803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:25,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:25,815 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:25,815 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:25,815 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:25,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743445804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:25,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:25,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:25,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:25,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:25,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743445913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:25,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:25,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:25,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:25,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:25,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743445924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:25,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:25,931 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:25,931 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:25,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1732743445924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:25,931 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:25,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:25,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743445924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:25,931 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:25,932 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:25,932 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:25,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:25,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:25,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743445924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:25,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:25,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:25,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:25,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:25,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:26,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:26,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743446133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:26,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:26,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:26,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:26,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:26,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743446144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:26,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:26,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:26,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:26,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1732743446144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:26,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:26,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:26,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743446144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:26,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:26,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:26,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:26,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743446144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:26,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:26,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:26,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:26,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:26,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:26,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:26,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:26,177 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/21d92481ae3c454e824c0ac32965be98 2024-11-27T21:36:26,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/e0273bb320304aa0920b1f4627e20817 is 50, key is test_row_0/C:col10/1732743384599/Put/seqid=0 2024-11-27T21:36:26,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742286_1462 (size=12001) 2024-11-27T21:36:26,195 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/e0273bb320304aa0920b1f4627e20817 2024-11-27T21:36:26,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/2919a52cf0774516acd56aba31cbbf9f as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/2919a52cf0774516acd56aba31cbbf9f 2024-11-27T21:36:26,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/2919a52cf0774516acd56aba31cbbf9f, entries=150, sequenceid=50, filesize=11.7 K 2024-11-27T21:36:26,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/21d92481ae3c454e824c0ac32965be98 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/21d92481ae3c454e824c0ac32965be98 2024-11-27T21:36:26,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/21d92481ae3c454e824c0ac32965be98, entries=150, sequenceid=50, filesize=11.7 K 2024-11-27T21:36:26,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/e0273bb320304aa0920b1f4627e20817 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/e0273bb320304aa0920b1f4627e20817 2024-11-27T21:36:26,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/e0273bb320304aa0920b1f4627e20817, entries=150, sequenceid=50, filesize=11.7 K 2024-11-27T21:36:26,215 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for b39500f7a69456fd382cec5091ca715b in 477ms, sequenceid=50, compaction requested=true 2024-11-27T21:36:26,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:26,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b39500f7a69456fd382cec5091ca715b:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:36:26,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:26,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b39500f7a69456fd382cec5091ca715b:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:36:26,215 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:26,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:26,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b39500f7a69456fd382cec5091ca715b:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:36:26,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:36:26,215 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:26,216 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38339 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:26,216 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): b39500f7a69456fd382cec5091ca715b/A is initiating minor compaction (all files) 2024-11-27T21:36:26,216 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:26,216 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b39500f7a69456fd382cec5091ca715b/A in TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:26,216 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): b39500f7a69456fd382cec5091ca715b/B is initiating minor compaction (all files) 2024-11-27T21:36:26,216 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b39500f7a69456fd382cec5091ca715b/B in TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:26,216 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/e2cb3d5f3fa14aba965026037e408ebb, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/98468e9e0ea0413187b3afc43e788dfe, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/2919a52cf0774516acd56aba31cbbf9f] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp, totalSize=37.4 K 2024-11-27T21:36:26,216 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/317838de3d024c6fb7b347da2c3899e4, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/6f89273aa73d4120bf349574fdb8324a, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/21d92481ae3c454e824c0ac32965be98] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp, totalSize=32.9 K 2024-11-27T21:36:26,216 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 317838de3d024c6fb7b347da2c3899e4, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732743383395 2024-11-27T21:36:26,216 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2cb3d5f3fa14aba965026037e408ebb, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732743383395 2024-11-27T21:36:26,217 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f89273aa73d4120bf349574fdb8324a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732743383425 2024-11-27T21:36:26,217 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98468e9e0ea0413187b3afc43e788dfe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732743383425 2024-11-27T21:36:26,217 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2919a52cf0774516acd56aba31cbbf9f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732743384586 2024-11-27T21:36:26,217 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 21d92481ae3c454e824c0ac32965be98, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732743384586 2024-11-27T21:36:26,223 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b39500f7a69456fd382cec5091ca715b#B#compaction#388 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:26,224 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/b438ad892bf549819e7028a6b8a6e5cf is 50, key is test_row_0/B:col10/1732743384599/Put/seqid=0 2024-11-27T21:36:26,226 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b39500f7a69456fd382cec5091ca715b#A#compaction#389 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:26,226 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/b579b6e8655c4ffaae3b3cf6b4468bf2 is 50, key is test_row_0/A:col10/1732743384599/Put/seqid=0 2024-11-27T21:36:26,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742287_1463 (size=12104) 2024-11-27T21:36:26,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742288_1464 (size=12104) 2024-11-27T21:36:26,245 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/b579b6e8655c4ffaae3b3cf6b4468bf2 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/b579b6e8655c4ffaae3b3cf6b4468bf2 2024-11-27T21:36:26,251 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b39500f7a69456fd382cec5091ca715b/A of b39500f7a69456fd382cec5091ca715b into b579b6e8655c4ffaae3b3cf6b4468bf2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:26,251 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:26,251 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., storeName=b39500f7a69456fd382cec5091ca715b/A, priority=13, startTime=1732743386215; duration=0sec 2024-11-27T21:36:26,251 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:36:26,251 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b39500f7a69456fd382cec5091ca715b:A 2024-11-27T21:36:26,251 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:26,252 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:26,252 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): b39500f7a69456fd382cec5091ca715b/C is initiating minor compaction (all files) 2024-11-27T21:36:26,252 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b39500f7a69456fd382cec5091ca715b/C in TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:26,252 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/3ccd53ef3fe140d699a6a39b94d0be2c, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/75824f2e34f94ddd860ef72b3a526371, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/e0273bb320304aa0920b1f4627e20817] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp, totalSize=32.9 K 2024-11-27T21:36:26,253 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ccd53ef3fe140d699a6a39b94d0be2c, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732743383395 2024-11-27T21:36:26,253 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75824f2e34f94ddd860ef72b3a526371, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732743383425 2024-11-27T21:36:26,253 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0273bb320304aa0920b1f4627e20817, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732743384586 2024-11-27T21:36:26,259 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b39500f7a69456fd382cec5091ca715b#C#compaction#390 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:26,260 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/b104e9f34ae24745b9672ba32d498d73 is 50, key is test_row_0/C:col10/1732743384599/Put/seqid=0 2024-11-27T21:36:26,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742289_1465 (size=12104) 2024-11-27T21:36:26,275 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/b104e9f34ae24745b9672ba32d498d73 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/b104e9f34ae24745b9672ba32d498d73 2024-11-27T21:36:26,279 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b39500f7a69456fd382cec5091ca715b/C of b39500f7a69456fd382cec5091ca715b into b104e9f34ae24745b9672ba32d498d73(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:26,279 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:26,279 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., storeName=b39500f7a69456fd382cec5091ca715b/C, priority=13, startTime=1732743386215; duration=0sec 2024-11-27T21:36:26,279 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:26,279 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b39500f7a69456fd382cec5091ca715b:C 2024-11-27T21:36:26,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on b39500f7a69456fd382cec5091ca715b 2024-11-27T21:36:26,460 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:36:26,460 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b39500f7a69456fd382cec5091ca715b 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-27T21:36:26,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/e53fed26c2774073a4e0547711e12cee is 50, key is test_row_0/A:col10/1732743385800/Put/seqid=0 2024-11-27T21:36:26,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742290_1466 (size=14341) 2024-11-27T21:36:26,469 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/e53fed26c2774073a4e0547711e12cee 2024-11-27T21:36:26,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/c6f3f1fb449448b9abffee13222d0c8e is 50, key is test_row_0/B:col10/1732743385800/Put/seqid=0 2024-11-27T21:36:26,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742291_1467 (size=12001) 2024-11-27T21:36:26,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:26,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743446482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:26,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:26,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:26,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:26,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:26,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1732743446484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:26,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:26,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:26,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:26,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:26,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743446484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:26,496 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:26,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:26,496 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:26,496 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:26,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743446485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:26,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:26,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:26,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:26,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:26,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743446486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:26,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:26,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:26,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:26,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:26,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743446593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:26,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:26,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:26,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:26,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:26,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1732743446604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:26,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:26,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743446604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:26,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:26,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:26,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:26,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:26,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743446604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:26,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:26,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:26,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:26,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:26,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:26,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:26,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:26,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743446604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:26,612 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:26,612 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:26,612 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:26,641 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/b438ad892bf549819e7028a6b8a6e5cf as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/b438ad892bf549819e7028a6b8a6e5cf 2024-11-27T21:36:26,645 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b39500f7a69456fd382cec5091ca715b/B of b39500f7a69456fd382cec5091ca715b into b438ad892bf549819e7028a6b8a6e5cf(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:26,646 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:26,646 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., storeName=b39500f7a69456fd382cec5091ca715b/B, priority=13, startTime=1732743386215; duration=0sec 2024-11-27T21:36:26,646 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:26,646 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b39500f7a69456fd382cec5091ca715b:B 2024-11-27T21:36:26,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:26,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743446814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:26,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:26,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:26,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:26,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:26,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1732743446814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:26,822 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:26,822 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:26,822 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:26,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:26,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743446823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:26,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:26,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:26,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743446823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:26,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:26,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:26,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:26,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743446823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:26,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:26,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:26,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:26,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:26,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:26,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:26,880 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/c6f3f1fb449448b9abffee13222d0c8e 2024-11-27T21:36:26,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/34cf18dcd2994579a44f7cbcacedefe2 is 50, key is test_row_0/C:col10/1732743385800/Put/seqid=0 2024-11-27T21:36:26,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742292_1468 (size=12001) 2024-11-27T21:36:27,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743447133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:27,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1732743447134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:27,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:27,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:27,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:27,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:27,139 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743447134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:27,139 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:27,139 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:27,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:27,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:27,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:27,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743447134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:27,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743447134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:27,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:27,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:27,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:27,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:27,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:27,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:27,292 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/34cf18dcd2994579a44f7cbcacedefe2 2024-11-27T21:36:27,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/e53fed26c2774073a4e0547711e12cee as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/e53fed26c2774073a4e0547711e12cee 2024-11-27T21:36:27,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/e53fed26c2774073a4e0547711e12cee, entries=200, sequenceid=76, filesize=14.0 K 2024-11-27T21:36:27,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/c6f3f1fb449448b9abffee13222d0c8e as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/c6f3f1fb449448b9abffee13222d0c8e 2024-11-27T21:36:27,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/c6f3f1fb449448b9abffee13222d0c8e, entries=150, sequenceid=76, filesize=11.7 K 2024-11-27T21:36:27,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/34cf18dcd2994579a44f7cbcacedefe2 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/34cf18dcd2994579a44f7cbcacedefe2 2024-11-27T21:36:27,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/34cf18dcd2994579a44f7cbcacedefe2, entries=150, sequenceid=76, filesize=11.7 K 2024-11-27T21:36:27,307 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for b39500f7a69456fd382cec5091ca715b in 847ms, sequenceid=76, compaction requested=false 2024-11-27T21:36:27,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:27,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-27T21:36:27,553 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:36:27,554 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:36:27,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-27T21:36:27,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-27T21:36:27,555 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:36:27,556 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:36:27,556 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:36:27,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on b39500f7a69456fd382cec5091ca715b 2024-11-27T21:36:27,649 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:36:27,649 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b39500f7a69456fd382cec5091ca715b 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-27T21:36:27,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/75f46c98d4ad4c18a21f6364b4eb0585 is 50, key is test_row_0/A:col10/1732743387648/Put/seqid=0 2024-11-27T21:36:27,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742293_1469 (size=14341) 2024-11-27T21:36:27,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-27T21:36:27,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:27,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743447693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:27,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:27,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743447694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:27,705 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:27,705 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:27,705 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:27,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:27,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:27,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:27,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:27,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743447695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:27,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:27,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:27,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:27,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:27,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743447696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:27,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:27,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:27,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:27,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:27,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1732743447697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:27,708 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:27,708 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:27,708 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:27,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-27T21:36:27,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:27,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:27,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:27,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:27,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:27,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:27,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:27,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743447814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:27,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:27,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:27,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:27,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:27,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743447814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:27,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:27,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:27,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:27,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:27,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743447814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:27,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:27,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:27,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:27,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:27,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743447814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:27,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:27,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:27,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:27,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:27,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1732743447814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:27,820 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:27,820 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:27,820 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:27,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-27T21:36:27,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:27,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:27,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:27,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:27,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:27,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:27,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-27T21:36:28,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-27T21:36:28,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:28,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:28,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:28,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:28,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:28,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:28,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:28,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743448023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:28,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:28,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743448023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:28,027 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:28,027 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:28,027 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:28,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:28,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743448023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:28,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:28,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:28,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:28,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:28,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:28,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:28,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743448023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:28,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:28,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:28,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:28,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:28,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:28,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1732743448023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:28,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:28,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:28,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:28,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/75f46c98d4ad4c18a21f6364b4eb0585 2024-11-27T21:36:28,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/bd782bc2ac7549498540b7a79e590965 is 50, key is test_row_0/B:col10/1732743387648/Put/seqid=0 2024-11-27T21:36:28,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742294_1470 (size=12001) 2024-11-27T21:36:28,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-27T21:36:28,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:28,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:28,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:28,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:28,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:28,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:28,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-27T21:36:28,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-27T21:36:28,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:28,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:28,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:28,321 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:28,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:28,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:28,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:28,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743448334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:28,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:28,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743448334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:28,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:28,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:28,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:28,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:28,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:28,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743448334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:28,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:28,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:28,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:28,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:28,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:28,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743448334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:28,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:28,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:28,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:28,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1732743448334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:28,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:28,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:28,342 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:28,342 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:28,342 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:28,470 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/bd782bc2ac7549498540b7a79e590965 2024-11-27T21:36:28,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-27T21:36:28,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:28,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:28,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:28,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:28,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:28,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:28,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/0ca31d296cbe4f0f89b6f830f655a26c is 50, key is test_row_0/C:col10/1732743387648/Put/seqid=0 2024-11-27T21:36:28,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742295_1471 (size=12001) 2024-11-27T21:36:28,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/0ca31d296cbe4f0f89b6f830f655a26c 2024-11-27T21:36:28,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/75f46c98d4ad4c18a21f6364b4eb0585 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/75f46c98d4ad4c18a21f6364b4eb0585 2024-11-27T21:36:28,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/75f46c98d4ad4c18a21f6364b4eb0585, entries=200, sequenceid=91, filesize=14.0 K 2024-11-27T21:36:28,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/bd782bc2ac7549498540b7a79e590965 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/bd782bc2ac7549498540b7a79e590965 2024-11-27T21:36:28,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/bd782bc2ac7549498540b7a79e590965, entries=150, sequenceid=91, filesize=11.7 K 2024-11-27T21:36:28,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/0ca31d296cbe4f0f89b6f830f655a26c as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/0ca31d296cbe4f0f89b6f830f655a26c 2024-11-27T21:36:28,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/0ca31d296cbe4f0f89b6f830f655a26c, entries=150, sequenceid=91, filesize=11.7 K 2024-11-27T21:36:28,501 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for b39500f7a69456fd382cec5091ca715b in 852ms, sequenceid=91, compaction requested=true 2024-11-27T21:36:28,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:28,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b39500f7a69456fd382cec5091ca715b:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:36:28,501 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:28,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:28,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b39500f7a69456fd382cec5091ca715b:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:36:28,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:28,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b39500f7a69456fd382cec5091ca715b:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:36:28,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:36:28,502 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:28,503 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:28,503 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): b39500f7a69456fd382cec5091ca715b/A is initiating minor compaction (all files) 2024-11-27T21:36:28,503 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b39500f7a69456fd382cec5091ca715b/A in TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:28,503 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/b579b6e8655c4ffaae3b3cf6b4468bf2, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/e53fed26c2774073a4e0547711e12cee, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/75f46c98d4ad4c18a21f6364b4eb0585] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp, totalSize=39.8 K 2024-11-27T21:36:28,504 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:28,504 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting b579b6e8655c4ffaae3b3cf6b4468bf2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732743384586 2024-11-27T21:36:28,504 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): b39500f7a69456fd382cec5091ca715b/B is initiating minor compaction (all files) 2024-11-27T21:36:28,504 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b39500f7a69456fd382cec5091ca715b/B in TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:28,504 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/b438ad892bf549819e7028a6b8a6e5cf, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/c6f3f1fb449448b9abffee13222d0c8e, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/bd782bc2ac7549498540b7a79e590965] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp, totalSize=35.3 K 2024-11-27T21:36:28,505 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting e53fed26c2774073a4e0547711e12cee, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732743385800 2024-11-27T21:36:28,505 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting b438ad892bf549819e7028a6b8a6e5cf, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732743384586 2024-11-27T21:36:28,505 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75f46c98d4ad4c18a21f6364b4eb0585, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732743386482 2024-11-27T21:36:28,505 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting c6f3f1fb449448b9abffee13222d0c8e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732743385800 2024-11-27T21:36:28,505 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting bd782bc2ac7549498540b7a79e590965, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732743386483 2024-11-27T21:36:28,512 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b39500f7a69456fd382cec5091ca715b#B#compaction#397 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:28,513 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/a90203520a384fada847094c52d14e7d is 50, key is test_row_0/B:col10/1732743387648/Put/seqid=0 2024-11-27T21:36:28,514 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b39500f7a69456fd382cec5091ca715b#A#compaction#398 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:28,515 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/1300846a9e154202b504b26f03cf9c1d is 50, key is test_row_0/A:col10/1732743387648/Put/seqid=0 2024-11-27T21:36:28,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742296_1472 (size=12207) 2024-11-27T21:36:28,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742297_1473 (size=12207) 2024-11-27T21:36:28,525 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/1300846a9e154202b504b26f03cf9c1d as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/1300846a9e154202b504b26f03cf9c1d 2024-11-27T21:36:28,529 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b39500f7a69456fd382cec5091ca715b/A of b39500f7a69456fd382cec5091ca715b into 1300846a9e154202b504b26f03cf9c1d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:28,529 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:28,529 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., storeName=b39500f7a69456fd382cec5091ca715b/A, priority=13, startTime=1732743388501; duration=0sec 2024-11-27T21:36:28,529 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:36:28,529 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b39500f7a69456fd382cec5091ca715b:A 2024-11-27T21:36:28,529 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:28,530 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:28,530 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): b39500f7a69456fd382cec5091ca715b/C is initiating minor compaction (all files) 2024-11-27T21:36:28,530 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b39500f7a69456fd382cec5091ca715b/C in TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:28,530 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/b104e9f34ae24745b9672ba32d498d73, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/34cf18dcd2994579a44f7cbcacedefe2, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/0ca31d296cbe4f0f89b6f830f655a26c] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp, totalSize=35.3 K 2024-11-27T21:36:28,530 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting b104e9f34ae24745b9672ba32d498d73, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732743384586 2024-11-27T21:36:28,531 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34cf18dcd2994579a44f7cbcacedefe2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732743385800 2024-11-27T21:36:28,531 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ca31d296cbe4f0f89b6f830f655a26c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732743386483 2024-11-27T21:36:28,536 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b39500f7a69456fd382cec5091ca715b#C#compaction#399 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:28,536 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/82417af6b37d4df7ad217871d8deb194 is 50, key is test_row_0/C:col10/1732743387648/Put/seqid=0 2024-11-27T21:36:28,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742298_1474 (size=12207) 2024-11-27T21:36:28,549 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/82417af6b37d4df7ad217871d8deb194 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/82417af6b37d4df7ad217871d8deb194 2024-11-27T21:36:28,553 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b39500f7a69456fd382cec5091ca715b/C of b39500f7a69456fd382cec5091ca715b into 82417af6b37d4df7ad217871d8deb194(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:28,553 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:28,553 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., storeName=b39500f7a69456fd382cec5091ca715b/C, priority=13, startTime=1732743388502; duration=0sec 2024-11-27T21:36:28,553 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:28,553 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b39500f7a69456fd382cec5091ca715b:C 2024-11-27T21:36:28,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-27T21:36:28,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:28,626 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing b39500f7a69456fd382cec5091ca715b 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-27T21:36:28,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/bf4732f2e7e64ec0ba54b960976f0a4a is 50, key is test_row_0/A:col10/1732743387693/Put/seqid=0 2024-11-27T21:36:28,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742299_1475 (size=12001) 2024-11-27T21:36:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-27T21:36:28,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on b39500f7a69456fd382cec5091ca715b 2024-11-27T21:36:28,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:28,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:28,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743448875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:28,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:28,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743448875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:28,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:28,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:28,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:28,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:28,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:28,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:28,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:28,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743448875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:28,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:28,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:28,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:28,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:28,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743448877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:28,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:28,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:28,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:28,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:28,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1732743448877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:28,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:28,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:28,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:28,923 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/a90203520a384fada847094c52d14e7d as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/a90203520a384fada847094c52d14e7d 2024-11-27T21:36:28,926 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b39500f7a69456fd382cec5091ca715b/B of b39500f7a69456fd382cec5091ca715b into a90203520a384fada847094c52d14e7d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:28,927 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:28,927 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., storeName=b39500f7a69456fd382cec5091ca715b/B, priority=13, startTime=1732743388502; duration=0sec 2024-11-27T21:36:28,927 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:28,927 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b39500f7a69456fd382cec5091ca715b:B 2024-11-27T21:36:28,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:28,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743448984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:28,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:28,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:28,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743448984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:28,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:28,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:28,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:28,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:28,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743448984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:28,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:28,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:28,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:28,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:28,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:29,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:29,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743448993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:29,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:29,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:29,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:29,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:29,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1732743448993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:29,002 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:29,002 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:29,002 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:29,038 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/bf4732f2e7e64ec0ba54b960976f0a4a 2024-11-27T21:36:29,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/ae0687cc309846c4ad995dd3cc4f42e5 is 50, key is test_row_0/B:col10/1732743387693/Put/seqid=0 2024-11-27T21:36:29,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742300_1476 (size=12001) 2024-11-27T21:36:29,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:29,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743449193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:29,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:29,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743449193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:29,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:29,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:29,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:29,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:29,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:29,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743449193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:29,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:29,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:29,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:29,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:29,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:29,204 WARN [regionserver/4585c39bed03:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 2, running: 1 2024-11-27T21:36:29,205 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/e2cb3d5f3fa14aba965026037e408ebb, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/98468e9e0ea0413187b3afc43e788dfe, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/b579b6e8655c4ffaae3b3cf6b4468bf2, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/2919a52cf0774516acd56aba31cbbf9f, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/e53fed26c2774073a4e0547711e12cee, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/75f46c98d4ad4c18a21f6364b4eb0585] to archive 2024-11-27T21:36:29,206 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T21:36:29,207 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/e2cb3d5f3fa14aba965026037e408ebb to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/e2cb3d5f3fa14aba965026037e408ebb 2024-11-27T21:36:29,208 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/98468e9e0ea0413187b3afc43e788dfe to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/98468e9e0ea0413187b3afc43e788dfe 2024-11-27T21:36:29,209 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/b579b6e8655c4ffaae3b3cf6b4468bf2 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/b579b6e8655c4ffaae3b3cf6b4468bf2 2024-11-27T21:36:29,210 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/2919a52cf0774516acd56aba31cbbf9f to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/2919a52cf0774516acd56aba31cbbf9f 2024-11-27T21:36:29,211 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/e53fed26c2774073a4e0547711e12cee to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/e53fed26c2774073a4e0547711e12cee 2024-11-27T21:36:29,212 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/75f46c98d4ad4c18a21f6364b4eb0585 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/75f46c98d4ad4c18a21f6364b4eb0585 2024-11-27T21:36:29,213 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/317838de3d024c6fb7b347da2c3899e4, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/6f89273aa73d4120bf349574fdb8324a, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/b438ad892bf549819e7028a6b8a6e5cf, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/21d92481ae3c454e824c0ac32965be98, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/c6f3f1fb449448b9abffee13222d0c8e, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/bd782bc2ac7549498540b7a79e590965] to archive 2024-11-27T21:36:29,214 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T21:36:29,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:29,215 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/317838de3d024c6fb7b347da2c3899e4 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/317838de3d024c6fb7b347da2c3899e4 2024-11-27T21:36:29,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743449213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:29,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:29,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:29,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:29,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:29,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1732743449213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:29,215 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:29,216 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:29,216 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:29,216 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/6f89273aa73d4120bf349574fdb8324a to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/6f89273aa73d4120bf349574fdb8324a 2024-11-27T21:36:29,217 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/b438ad892bf549819e7028a6b8a6e5cf to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/b438ad892bf549819e7028a6b8a6e5cf 2024-11-27T21:36:29,218 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/21d92481ae3c454e824c0ac32965be98 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/21d92481ae3c454e824c0ac32965be98 2024-11-27T21:36:29,219 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/c6f3f1fb449448b9abffee13222d0c8e to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/c6f3f1fb449448b9abffee13222d0c8e 2024-11-27T21:36:29,220 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/bd782bc2ac7549498540b7a79e590965 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/bd782bc2ac7549498540b7a79e590965 2024-11-27T21:36:29,222 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/3ccd53ef3fe140d699a6a39b94d0be2c, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/75824f2e34f94ddd860ef72b3a526371, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/b104e9f34ae24745b9672ba32d498d73, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/e0273bb320304aa0920b1f4627e20817, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/34cf18dcd2994579a44f7cbcacedefe2, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/0ca31d296cbe4f0f89b6f830f655a26c] to archive 2024-11-27T21:36:29,223 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T21:36:29,224 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/3ccd53ef3fe140d699a6a39b94d0be2c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/3ccd53ef3fe140d699a6a39b94d0be2c 2024-11-27T21:36:29,225 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/75824f2e34f94ddd860ef72b3a526371 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/75824f2e34f94ddd860ef72b3a526371 2024-11-27T21:36:29,226 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/b104e9f34ae24745b9672ba32d498d73 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/b104e9f34ae24745b9672ba32d498d73 2024-11-27T21:36:29,227 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/e0273bb320304aa0920b1f4627e20817 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/e0273bb320304aa0920b1f4627e20817 2024-11-27T21:36:29,228 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/34cf18dcd2994579a44f7cbcacedefe2 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/34cf18dcd2994579a44f7cbcacedefe2 2024-11-27T21:36:29,229 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/0ca31d296cbe4f0f89b6f830f655a26c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/0ca31d296cbe4f0f89b6f830f655a26c 2024-11-27T21:36:29,455 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/ae0687cc309846c4ad995dd3cc4f42e5 2024-11-27T21:36:29,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/389e4cfe0b444c1aa46ceb3ac89f841c is 50, key is test_row_0/C:col10/1732743387693/Put/seqid=0 2024-11-27T21:36:29,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742301_1477 (size=12001) 2024-11-27T21:36:29,467 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/389e4cfe0b444c1aa46ceb3ac89f841c 2024-11-27T21:36:29,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/bf4732f2e7e64ec0ba54b960976f0a4a as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/bf4732f2e7e64ec0ba54b960976f0a4a 2024-11-27T21:36:29,474 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/bf4732f2e7e64ec0ba54b960976f0a4a, entries=150, sequenceid=115, filesize=11.7 K 2024-11-27T21:36:29,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/ae0687cc309846c4ad995dd3cc4f42e5 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/ae0687cc309846c4ad995dd3cc4f42e5 2024-11-27T21:36:29,477 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/ae0687cc309846c4ad995dd3cc4f42e5, entries=150, sequenceid=115, filesize=11.7 K 2024-11-27T21:36:29,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/389e4cfe0b444c1aa46ceb3ac89f841c as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/389e4cfe0b444c1aa46ceb3ac89f841c 2024-11-27T21:36:29,488 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/389e4cfe0b444c1aa46ceb3ac89f841c, entries=150, sequenceid=115, filesize=11.7 K 2024-11-27T21:36:29,489 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for b39500f7a69456fd382cec5091ca715b in 863ms, sequenceid=115, compaction requested=false 2024-11-27T21:36:29,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:29,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:29,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-27T21:36:29,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-27T21:36:29,492 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-27T21:36:29,492 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9340 sec 2024-11-27T21:36:29,493 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.9380 sec 2024-11-27T21:36:29,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on b39500f7a69456fd382cec5091ca715b 2024-11-27T21:36:29,521 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:36:29,521 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b39500f7a69456fd382cec5091ca715b 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-27T21:36:29,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/1f66fa5728d641b585ffa51789513f45 is 50, key is test_row_0/A:col10/1732743388873/Put/seqid=0 2024-11-27T21:36:29,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742302_1478 (size=16831) 2024-11-27T21:36:29,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:29,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743449547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:29,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:29,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:29,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:29,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:29,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743449548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:29,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:29,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:29,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:29,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:29,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743449550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:29,559 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:29,559 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:29,559 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:29,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:29,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743449551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:29,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:29,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:29,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:29,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:29,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1732743449557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:29,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:29,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:29,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:29,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:29,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743449663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:29,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:29,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:29,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743449663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:29,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:29,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:29,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:29,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:29,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743449663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:29,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:29,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:29,671 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:29,671 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:29,671 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:29,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:29,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743449673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:29,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:29,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:29,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1732743449673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:29,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:29,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:29,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-27T21:36:29,703 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:36:29,704 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:36:29,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-27T21:36:29,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-27T21:36:29,705 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:36:29,706 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:36:29,706 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:36:29,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-27T21:36:29,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-27T21:36:29,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:29,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:29,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:29,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:29,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:29,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:29,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:29,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743449873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:29,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:29,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:29,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:29,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:29,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743449883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:29,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:29,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743449883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:29,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:29,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:29,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:29,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:29,892 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:29,892 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:29,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:29,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743449893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:29,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:29,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:29,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:29,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:29,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1732743449893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:29,904 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:29,904 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:29,904 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:29,938 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/1f66fa5728d641b585ffa51789513f45 2024-11-27T21:36:29,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/33d0a182aec2445c956242a7f988129f is 50, key is test_row_0/B:col10/1732743388873/Put/seqid=0 2024-11-27T21:36:29,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742303_1479 (size=12101) 2024-11-27T21:36:30,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-27T21:36:30,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:30,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:30,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:30,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:30,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:30,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:30,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-27T21:36:30,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-27T21:36:30,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:30,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:30,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:30,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:30,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:30,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:30,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:30,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743450184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:30,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:30,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:30,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:30,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:30,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743450193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:30,203 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:30,203 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:30,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:30,203 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:30,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743450193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:30,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:30,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:30,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:30,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:30,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743450213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:30,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:30,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:30,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:30,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1732743450214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:30,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:30,222 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:30,222 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:30,222 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:30,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-27T21:36:30,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:30,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:30,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:30,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:30,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:30,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:30,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-27T21:36:30,348 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/33d0a182aec2445c956242a7f988129f 2024-11-27T21:36:30,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/225d5440cfe241d7987ce9f285d973fa is 50, key is test_row_0/C:col10/1732743388873/Put/seqid=0 2024-11-27T21:36:30,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742304_1480 (size=12101) 2024-11-27T21:36:30,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-27T21:36:30,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:30,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:30,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:30,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:30,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:30,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:30,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-27T21:36:30,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:30,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:30,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:30,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:30,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:30,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:30,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:30,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743450704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:30,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:30,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743450713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:30,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:30,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743450713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:30,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:30,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:30,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:30,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743450723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:30,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:30,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:30,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:30,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:30,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1732743450733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:30,745 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:30,745 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:30,745 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:30,765 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/225d5440cfe241d7987ce9f285d973fa 2024-11-27T21:36:30,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/1f66fa5728d641b585ffa51789513f45 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/1f66fa5728d641b585ffa51789513f45 2024-11-27T21:36:30,772 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/1f66fa5728d641b585ffa51789513f45, entries=250, sequenceid=133, filesize=16.4 K 2024-11-27T21:36:30,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/33d0a182aec2445c956242a7f988129f as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/33d0a182aec2445c956242a7f988129f 2024-11-27T21:36:30,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-27T21:36:30,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:30,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:30,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:30,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:30,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:30,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:30,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/33d0a182aec2445c956242a7f988129f, entries=150, sequenceid=133, filesize=11.8 K 2024-11-27T21:36:30,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/225d5440cfe241d7987ce9f285d973fa as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/225d5440cfe241d7987ce9f285d973fa 2024-11-27T21:36:30,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/225d5440cfe241d7987ce9f285d973fa, entries=150, sequenceid=133, filesize=11.8 K 2024-11-27T21:36:30,785 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for b39500f7a69456fd382cec5091ca715b in 1264ms, sequenceid=133, compaction requested=true 2024-11-27T21:36:30,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:30,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b39500f7a69456fd382cec5091ca715b:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:36:30,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:30,785 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:30,785 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:30,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b39500f7a69456fd382cec5091ca715b:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:36:30,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:30,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b39500f7a69456fd382cec5091ca715b:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:36:30,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:36:30,786 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:30,786 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:30,787 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): b39500f7a69456fd382cec5091ca715b/B is initiating minor compaction (all files) 2024-11-27T21:36:30,787 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): b39500f7a69456fd382cec5091ca715b/A is initiating minor compaction (all files) 2024-11-27T21:36:30,787 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b39500f7a69456fd382cec5091ca715b/B in TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:30,787 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b39500f7a69456fd382cec5091ca715b/A in TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:30,787 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/a90203520a384fada847094c52d14e7d, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/ae0687cc309846c4ad995dd3cc4f42e5, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/33d0a182aec2445c956242a7f988129f] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp, totalSize=35.5 K 2024-11-27T21:36:30,787 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/1300846a9e154202b504b26f03cf9c1d, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/bf4732f2e7e64ec0ba54b960976f0a4a, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/1f66fa5728d641b585ffa51789513f45] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp, totalSize=40.1 K 2024-11-27T21:36:30,787 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting a90203520a384fada847094c52d14e7d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732743386483 2024-11-27T21:36:30,787 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1300846a9e154202b504b26f03cf9c1d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732743386483 2024-11-27T21:36:30,788 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf4732f2e7e64ec0ba54b960976f0a4a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732743387693 2024-11-27T21:36:30,788 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting ae0687cc309846c4ad995dd3cc4f42e5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732743387693 2024-11-27T21:36:30,788 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f66fa5728d641b585ffa51789513f45, keycount=250, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732743388873 2024-11-27T21:36:30,788 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 33d0a182aec2445c956242a7f988129f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732743388873 2024-11-27T21:36:30,794 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b39500f7a69456fd382cec5091ca715b#A#compaction#406 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:30,795 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/0978edafe3ae40959ae32c53c5fa6650 is 50, key is test_row_0/A:col10/1732743388873/Put/seqid=0 2024-11-27T21:36:30,797 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b39500f7a69456fd382cec5091ca715b#B#compaction#407 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:30,798 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/3d3c1c8850f2424cb72ef6012c9cded7 is 50, key is test_row_0/B:col10/1732743388873/Put/seqid=0 2024-11-27T21:36:30,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742306_1482 (size=12204) 2024-11-27T21:36:30,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742305_1481 (size=12204) 2024-11-27T21:36:30,819 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/3d3c1c8850f2424cb72ef6012c9cded7 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/3d3c1c8850f2424cb72ef6012c9cded7 2024-11-27T21:36:30,822 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/0978edafe3ae40959ae32c53c5fa6650 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/0978edafe3ae40959ae32c53c5fa6650 2024-11-27T21:36:30,827 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b39500f7a69456fd382cec5091ca715b/B of b39500f7a69456fd382cec5091ca715b into 3d3c1c8850f2424cb72ef6012c9cded7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:30,827 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:30,827 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., storeName=b39500f7a69456fd382cec5091ca715b/B, priority=13, startTime=1732743390785; duration=0sec 2024-11-27T21:36:30,827 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:36:30,827 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b39500f7a69456fd382cec5091ca715b:B 2024-11-27T21:36:30,827 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:30,830 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:30,830 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): b39500f7a69456fd382cec5091ca715b/C is initiating minor compaction (all files) 2024-11-27T21:36:30,830 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b39500f7a69456fd382cec5091ca715b/A of b39500f7a69456fd382cec5091ca715b into 0978edafe3ae40959ae32c53c5fa6650(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:30,830 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b39500f7a69456fd382cec5091ca715b/C in TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:30,830 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:30,830 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., storeName=b39500f7a69456fd382cec5091ca715b/A, priority=13, startTime=1732743390785; duration=0sec 2024-11-27T21:36:30,830 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/82417af6b37d4df7ad217871d8deb194, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/389e4cfe0b444c1aa46ceb3ac89f841c, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/225d5440cfe241d7987ce9f285d973fa] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp, totalSize=35.5 K 2024-11-27T21:36:30,830 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:30,830 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b39500f7a69456fd382cec5091ca715b:A 2024-11-27T21:36:30,830 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 82417af6b37d4df7ad217871d8deb194, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732743386483 2024-11-27T21:36:30,831 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 389e4cfe0b444c1aa46ceb3ac89f841c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732743387693 2024-11-27T21:36:30,831 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 225d5440cfe241d7987ce9f285d973fa, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732743388873 2024-11-27T21:36:30,840 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b39500f7a69456fd382cec5091ca715b#C#compaction#408 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:30,841 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/8f0018b80f2b476fae9a374515250390 is 50, key is test_row_0/C:col10/1732743388873/Put/seqid=0 2024-11-27T21:36:30,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-27T21:36:30,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742307_1483 (size=12204) 2024-11-27T21:36:30,853 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/8f0018b80f2b476fae9a374515250390 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/8f0018b80f2b476fae9a374515250390 2024-11-27T21:36:30,859 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b39500f7a69456fd382cec5091ca715b/C of b39500f7a69456fd382cec5091ca715b into 8f0018b80f2b476fae9a374515250390(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:30,859 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:30,859 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., storeName=b39500f7a69456fd382cec5091ca715b/C, priority=13, startTime=1732743390785; duration=0sec 2024-11-27T21:36:30,859 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:30,859 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b39500f7a69456fd382cec5091ca715b:C 2024-11-27T21:36:30,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-27T21:36:30,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:30,931 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing b39500f7a69456fd382cec5091ca715b 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-27T21:36:30,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/72febdd05ec8447b8c13f7cae7e43b83 is 50, key is test_row_0/A:col10/1732743389549/Put/seqid=0 2024-11-27T21:36:30,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742308_1484 (size=12151) 2024-11-27T21:36:31,340 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/72febdd05ec8447b8c13f7cae7e43b83 2024-11-27T21:36:31,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/78bfd9bf375c41d3a0ece5b78aab4337 is 50, key is test_row_0/B:col10/1732743389549/Put/seqid=0 2024-11-27T21:36:31,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742309_1485 (size=12151) 2024-11-27T21:36:31,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on b39500f7a69456fd382cec5091ca715b 2024-11-27T21:36:31,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:31,751 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/78bfd9bf375c41d3a0ece5b78aab4337 2024-11-27T21:36:31,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/5c57060483ca429c92fba104086eedbd is 50, key is test_row_0/C:col10/1732743389549/Put/seqid=0 2024-11-27T21:36:31,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742310_1486 (size=12151) 2024-11-27T21:36:31,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:31,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1732743451753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:31,763 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,39097,1732743267048 for 'test_row_1' in b39500f7a69456fd382cec5091ca715b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:36:31,763 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:31,763 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:31,763 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:31,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:31,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743451754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:31,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:31,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743451755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:31,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:31,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:31,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:31,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:31,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:31,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:31,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:31,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743451762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:31,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:31,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:31,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:31,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:31,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743451763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:31,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:31,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:31,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:31,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-27T21:36:31,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:31,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743451873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:31,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:31,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:31,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:31,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:31,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743451873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:31,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:31,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:31,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:31,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:31,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743451873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:31,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:31,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:31,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:31,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:31,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743451883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:31,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:31,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:31,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:32,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:32,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743452083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:32,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:32,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:32,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:32,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743452083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:32,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:32,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:32,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:32,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:32,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743452083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:32,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:32,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:32,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:32,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:32,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:32,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743452103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:32,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:32,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:32,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:32,161 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/5c57060483ca429c92fba104086eedbd 2024-11-27T21:36:32,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/72febdd05ec8447b8c13f7cae7e43b83 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/72febdd05ec8447b8c13f7cae7e43b83 2024-11-27T21:36:32,169 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/72febdd05ec8447b8c13f7cae7e43b83, entries=150, sequenceid=155, filesize=11.9 K 2024-11-27T21:36:32,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/78bfd9bf375c41d3a0ece5b78aab4337 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/78bfd9bf375c41d3a0ece5b78aab4337 2024-11-27T21:36:32,172 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/78bfd9bf375c41d3a0ece5b78aab4337, entries=150, sequenceid=155, filesize=11.9 K 2024-11-27T21:36:32,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/5c57060483ca429c92fba104086eedbd as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/5c57060483ca429c92fba104086eedbd 2024-11-27T21:36:32,176 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/5c57060483ca429c92fba104086eedbd, entries=150, sequenceid=155, filesize=11.9 K 2024-11-27T21:36:32,177 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for b39500f7a69456fd382cec5091ca715b in 1246ms, sequenceid=155, compaction requested=false 2024-11-27T21:36:32,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:32,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:32,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-27T21:36:32,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-27T21:36:32,180 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-27T21:36:32,180 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4720 sec 2024-11-27T21:36:32,181 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 2.4760 sec 2024-11-27T21:36:32,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on b39500f7a69456fd382cec5091ca715b 2024-11-27T21:36:32,409 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:36:32,409 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b39500f7a69456fd382cec5091ca715b 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-27T21:36:32,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/136f17a68a984159a242b7ade09408cc is 50, key is test_row_0/A:col10/1732743392408/Put/seqid=0 2024-11-27T21:36:32,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742311_1487 (size=14541) 2024-11-27T21:36:32,423 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/136f17a68a984159a242b7ade09408cc 2024-11-27T21:36:32,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/01046d8c4b0749e4b51e044f2d97013d is 50, key is test_row_0/B:col10/1732743392408/Put/seqid=0 2024-11-27T21:36:32,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742312_1488 (size=12151) 2024-11-27T21:36:32,439 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/01046d8c4b0749e4b51e044f2d97013d 2024-11-27T21:36:32,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/9b7fffbbe2f443ffaa10e7997c06d0ae is 50, key is test_row_0/C:col10/1732743392408/Put/seqid=0 2024-11-27T21:36:32,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742313_1489 (size=12151) 2024-11-27T21:36:32,456 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/9b7fffbbe2f443ffaa10e7997c06d0ae 2024-11-27T21:36:32,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:32,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743452446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:32,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:32,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:32,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:32,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/136f17a68a984159a242b7ade09408cc as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/136f17a68a984159a242b7ade09408cc 2024-11-27T21:36:32,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:32,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743452452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:32,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:32,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/136f17a68a984159a242b7ade09408cc, entries=200, sequenceid=173, filesize=14.2 K 2024-11-27T21:36:32,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743452454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:32,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:32,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743452455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:32,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:32,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:32,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:32,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/01046d8c4b0749e4b51e044f2d97013d as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/01046d8c4b0749e4b51e044f2d97013d 2024-11-27T21:36:32,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:32,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:32,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:32,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:32,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:32,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:32,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/01046d8c4b0749e4b51e044f2d97013d, entries=150, sequenceid=173, filesize=11.9 K 2024-11-27T21:36:32,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/9b7fffbbe2f443ffaa10e7997c06d0ae as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/9b7fffbbe2f443ffaa10e7997c06d0ae 2024-11-27T21:36:32,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/9b7fffbbe2f443ffaa10e7997c06d0ae, entries=150, sequenceid=173, filesize=11.9 K 2024-11-27T21:36:32,478 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for b39500f7a69456fd382cec5091ca715b in 69ms, sequenceid=173, compaction requested=true 2024-11-27T21:36:32,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:32,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b39500f7a69456fd382cec5091ca715b:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:36:32,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:32,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b39500f7a69456fd382cec5091ca715b:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:36:32,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:32,478 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:32,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b39500f7a69456fd382cec5091ca715b:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:36:32,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:36:32,478 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:32,479 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36506 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:32,479 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38896 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:32,479 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): b39500f7a69456fd382cec5091ca715b/B is initiating minor compaction (all files) 2024-11-27T21:36:32,479 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): b39500f7a69456fd382cec5091ca715b/A is initiating minor compaction (all files) 2024-11-27T21:36:32,479 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b39500f7a69456fd382cec5091ca715b/B in TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:32,479 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b39500f7a69456fd382cec5091ca715b/A in TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:32,479 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/3d3c1c8850f2424cb72ef6012c9cded7, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/78bfd9bf375c41d3a0ece5b78aab4337, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/01046d8c4b0749e4b51e044f2d97013d] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp, totalSize=35.7 K 2024-11-27T21:36:32,479 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/0978edafe3ae40959ae32c53c5fa6650, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/72febdd05ec8447b8c13f7cae7e43b83, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/136f17a68a984159a242b7ade09408cc] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp, totalSize=38.0 K 2024-11-27T21:36:32,480 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0978edafe3ae40959ae32c53c5fa6650, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732743388873 2024-11-27T21:36:32,480 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d3c1c8850f2424cb72ef6012c9cded7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732743388873 2024-11-27T21:36:32,481 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72febdd05ec8447b8c13f7cae7e43b83, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732743389544 2024-11-27T21:36:32,481 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 78bfd9bf375c41d3a0ece5b78aab4337, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732743389544 2024-11-27T21:36:32,481 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 01046d8c4b0749e4b51e044f2d97013d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732743391751 2024-11-27T21:36:32,481 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 136f17a68a984159a242b7ade09408cc, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732743391749 2024-11-27T21:36:32,489 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b39500f7a69456fd382cec5091ca715b#A#compaction#415 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:32,489 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b39500f7a69456fd382cec5091ca715b#B#compaction#416 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:32,490 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/788e4d09b4b64f3bbe8c4156f0c06f45 is 50, key is test_row_0/A:col10/1732743392408/Put/seqid=0 2024-11-27T21:36:32,490 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/eb6b298c0eb14de180aebae076a8a4d7 is 50, key is test_row_0/B:col10/1732743392408/Put/seqid=0 2024-11-27T21:36:32,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742315_1491 (size=12357) 2024-11-27T21:36:32,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742314_1490 (size=12357) 2024-11-27T21:36:32,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on b39500f7a69456fd382cec5091ca715b 2024-11-27T21:36:32,568 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:36:32,569 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b39500f7a69456fd382cec5091ca715b 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-27T21:36:32,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/6ab2c2d1767c485680b562167b7ea6f3 is 50, key is test_row_0/A:col10/1732743392445/Put/seqid=0 2024-11-27T21:36:32,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742316_1492 (size=14541) 2024-11-27T21:36:32,579 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/6ab2c2d1767c485680b562167b7ea6f3 2024-11-27T21:36:32,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/d5ab9bff5d354260bd69a5f7c15bd1a6 is 50, key is test_row_0/B:col10/1732743392445/Put/seqid=0 2024-11-27T21:36:32,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742317_1493 (size=12151) 2024-11-27T21:36:32,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:32,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743452606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:32,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:32,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:32,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:32,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:32,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743452608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:32,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:32,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:32,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:32,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:32,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743452608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:32,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:32,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:32,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:32,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:32,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743452609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:32,624 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:32,624 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:32,624 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:32,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:32,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743452713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:32,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:32,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:32,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:32,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:32,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743452733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:32,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:32,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:32,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:32,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:32,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743452733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:32,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:32,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:32,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:32,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:32,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743452733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:32,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:32,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:32,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:32,907 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/eb6b298c0eb14de180aebae076a8a4d7 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/eb6b298c0eb14de180aebae076a8a4d7 2024-11-27T21:36:32,908 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/788e4d09b4b64f3bbe8c4156f0c06f45 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/788e4d09b4b64f3bbe8c4156f0c06f45 2024-11-27T21:36:32,912 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b39500f7a69456fd382cec5091ca715b/B of b39500f7a69456fd382cec5091ca715b into eb6b298c0eb14de180aebae076a8a4d7(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:32,912 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:32,912 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., storeName=b39500f7a69456fd382cec5091ca715b/B, priority=13, startTime=1732743392478; duration=0sec 2024-11-27T21:36:32,912 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:36:32,912 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b39500f7a69456fd382cec5091ca715b:B 2024-11-27T21:36:32,912 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:32,917 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b39500f7a69456fd382cec5091ca715b/A of b39500f7a69456fd382cec5091ca715b into 788e4d09b4b64f3bbe8c4156f0c06f45(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:32,917 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:32,918 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., storeName=b39500f7a69456fd382cec5091ca715b/A, priority=13, startTime=1732743392478; duration=0sec 2024-11-27T21:36:32,918 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:32,918 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b39500f7a69456fd382cec5091ca715b:A 2024-11-27T21:36:32,918 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36506 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:32,918 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): b39500f7a69456fd382cec5091ca715b/C is initiating minor compaction (all files) 2024-11-27T21:36:32,918 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b39500f7a69456fd382cec5091ca715b/C in TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:32,918 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/8f0018b80f2b476fae9a374515250390, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/5c57060483ca429c92fba104086eedbd, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/9b7fffbbe2f443ffaa10e7997c06d0ae] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp, totalSize=35.7 K 2024-11-27T21:36:32,919 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f0018b80f2b476fae9a374515250390, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732743388873 2024-11-27T21:36:32,920 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c57060483ca429c92fba104086eedbd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732743389544 2024-11-27T21:36:32,920 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b7fffbbe2f443ffaa10e7997c06d0ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732743391751 2024-11-27T21:36:32,933 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b39500f7a69456fd382cec5091ca715b#C#compaction#419 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:32,934 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/45e60194b9074f328f6a5d2ebcb9057f is 50, key is test_row_0/C:col10/1732743392408/Put/seqid=0 2024-11-27T21:36:32,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:32,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743452923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:32,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:32,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:32,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:32,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742318_1494 (size=12357) 2024-11-27T21:36:32,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:32,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743452954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:32,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:32,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:32,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:32,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:32,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743452954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:32,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:32,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743452954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:32,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:32,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:32,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:32,957 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:32,957 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:32,957 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:32,989 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/d5ab9bff5d354260bd69a5f7c15bd1a6 2024-11-27T21:36:32,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/77981dbfb44f42a0b728938f5d5e2774 is 50, key is test_row_0/C:col10/1732743392445/Put/seqid=0 2024-11-27T21:36:33,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742319_1495 (size=12151) 2024-11-27T21:36:33,007 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/77981dbfb44f42a0b728938f5d5e2774 2024-11-27T21:36:33,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/6ab2c2d1767c485680b562167b7ea6f3 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/6ab2c2d1767c485680b562167b7ea6f3 2024-11-27T21:36:33,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/6ab2c2d1767c485680b562167b7ea6f3, entries=200, sequenceid=192, filesize=14.2 K 2024-11-27T21:36:33,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/d5ab9bff5d354260bd69a5f7c15bd1a6 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/d5ab9bff5d354260bd69a5f7c15bd1a6 2024-11-27T21:36:33,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/d5ab9bff5d354260bd69a5f7c15bd1a6, entries=150, sequenceid=192, filesize=11.9 K 2024-11-27T21:36:33,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/77981dbfb44f42a0b728938f5d5e2774 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/77981dbfb44f42a0b728938f5d5e2774 2024-11-27T21:36:33,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/77981dbfb44f42a0b728938f5d5e2774, entries=150, sequenceid=192, filesize=11.9 K 2024-11-27T21:36:33,030 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for b39500f7a69456fd382cec5091ca715b in 461ms, sequenceid=192, compaction requested=false 2024-11-27T21:36:33,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:33,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on b39500f7a69456fd382cec5091ca715b 2024-11-27T21:36:33,248 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:36:33,248 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b39500f7a69456fd382cec5091ca715b 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-27T21:36:33,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/378c98f42aae468e9eee9e5fce05a089 is 50, key is test_row_0/A:col10/1732743392606/Put/seqid=0 2024-11-27T21:36:33,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742320_1496 (size=14541) 2024-11-27T21:36:33,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:33,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743453288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:33,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:33,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:33,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743453289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:33,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:33,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:33,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:33,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:33,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:33,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:33,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743453300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:33,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:33,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:33,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:33,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:33,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743453302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:33,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:33,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:33,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:33,347 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/45e60194b9074f328f6a5d2ebcb9057f as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/45e60194b9074f328f6a5d2ebcb9057f 2024-11-27T21:36:33,351 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b39500f7a69456fd382cec5091ca715b/C of b39500f7a69456fd382cec5091ca715b into 45e60194b9074f328f6a5d2ebcb9057f(size=12.1 K), total size for store is 23.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:33,351 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:33,351 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., storeName=b39500f7a69456fd382cec5091ca715b/C, priority=13, startTime=1732743392478; duration=0sec 2024-11-27T21:36:33,351 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:33,351 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b39500f7a69456fd382cec5091ca715b:C 2024-11-27T21:36:33,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:33,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743453413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:33,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:33,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743453413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:33,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:33,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:33,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:33,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:33,423 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:33,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:33,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:33,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743453414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:33,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:33,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:33,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:33,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:33,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743453423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:33,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:33,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:33,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:33,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:33,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743453633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:33,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:33,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:33,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:33,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:33,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743453634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:33,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:33,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:33,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:33,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:33,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743453634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:33,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:33,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:33,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:33,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:33,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743453634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:33,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:33,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:33,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:33,658 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/378c98f42aae468e9eee9e5fce05a089 2024-11-27T21:36:33,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/c575e90fea5346f4ad41769b459cb5b6 is 50, key is test_row_0/B:col10/1732743392606/Put/seqid=0 2024-11-27T21:36:33,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742321_1497 (size=12151) 2024-11-27T21:36:33,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:33,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1732743453783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:33,793 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,39097,1732743267048 for 'test_row_1' in b39500f7a69456fd382cec5091ca715b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:36:33,794 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:33,794 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:33,794 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:33,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-27T21:36:33,873 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:36:33,874 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:36:33,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-27T21:36:33,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-27T21:36:33,875 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:36:33,876 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:36:33,876 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:36:33,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:33,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743453953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:33,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:33,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743453954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:33,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:33,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:33,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:33,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:33,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:33,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743453954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:33,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:33,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:33,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:33,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:33,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:33,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743453954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:33,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:33,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:33,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:33,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:33,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-27T21:36:34,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-27T21:36:34,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:34,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:34,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:34,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:34,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:34,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:34,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/c575e90fea5346f4ad41769b459cb5b6 2024-11-27T21:36:34,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/2e663844ed264e3993c9ad55a44d35c3 is 50, key is test_row_0/C:col10/1732743392606/Put/seqid=0 2024-11-27T21:36:34,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742322_1498 (size=12151) 2024-11-27T21:36:34,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-27T21:36:34,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:34,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:34,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:34,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:34,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:34,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:34,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-27T21:36:34,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-27T21:36:34,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:34,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:34,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:34,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:34,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:34,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:34,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:34,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743454463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:34,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:34,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:34,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:34,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:34,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743454473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:34,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:34,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:34,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743454473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:34,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:34,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:34,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:34,480 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:34,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743454473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:34,480 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:34,480 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:34,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:34,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:34,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:34,483 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/2e663844ed264e3993c9ad55a44d35c3 2024-11-27T21:36:34,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/378c98f42aae468e9eee9e5fce05a089 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/378c98f42aae468e9eee9e5fce05a089 2024-11-27T21:36:34,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-27T21:36:34,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:34,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:34,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:34,488 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:34,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:34,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:34,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/378c98f42aae468e9eee9e5fce05a089, entries=200, sequenceid=212, filesize=14.2 K 2024-11-27T21:36:34,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/c575e90fea5346f4ad41769b459cb5b6 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/c575e90fea5346f4ad41769b459cb5b6 2024-11-27T21:36:34,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/c575e90fea5346f4ad41769b459cb5b6, entries=150, sequenceid=212, filesize=11.9 K 2024-11-27T21:36:34,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/2e663844ed264e3993c9ad55a44d35c3 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/2e663844ed264e3993c9ad55a44d35c3 2024-11-27T21:36:34,498 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/2e663844ed264e3993c9ad55a44d35c3, entries=150, sequenceid=212, filesize=11.9 K 2024-11-27T21:36:34,499 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for b39500f7a69456fd382cec5091ca715b in 1251ms, sequenceid=212, compaction requested=true 2024-11-27T21:36:34,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:34,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b39500f7a69456fd382cec5091ca715b:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:36:34,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:34,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b39500f7a69456fd382cec5091ca715b:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:36:34,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:34,499 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:34,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b39500f7a69456fd382cec5091ca715b:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:36:34,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:36:34,499 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:34,500 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:34,500 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41439 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:34,500 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): b39500f7a69456fd382cec5091ca715b/A is initiating minor compaction (all files) 2024-11-27T21:36:34,500 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): b39500f7a69456fd382cec5091ca715b/B is initiating minor compaction (all files) 2024-11-27T21:36:34,500 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b39500f7a69456fd382cec5091ca715b/B in TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:34,500 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b39500f7a69456fd382cec5091ca715b/A in TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:34,500 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/788e4d09b4b64f3bbe8c4156f0c06f45, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/6ab2c2d1767c485680b562167b7ea6f3, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/378c98f42aae468e9eee9e5fce05a089] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp, totalSize=40.5 K 2024-11-27T21:36:34,500 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/eb6b298c0eb14de180aebae076a8a4d7, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/d5ab9bff5d354260bd69a5f7c15bd1a6, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/c575e90fea5346f4ad41769b459cb5b6] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp, totalSize=35.8 K 2024-11-27T21:36:34,500 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 788e4d09b4b64f3bbe8c4156f0c06f45, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732743391751 2024-11-27T21:36:34,500 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting eb6b298c0eb14de180aebae076a8a4d7, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732743391751 2024-11-27T21:36:34,501 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ab2c2d1767c485680b562167b7ea6f3, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732743392445 2024-11-27T21:36:34,501 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting d5ab9bff5d354260bd69a5f7c15bd1a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732743392445 2024-11-27T21:36:34,501 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 378c98f42aae468e9eee9e5fce05a089, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732743392606 2024-11-27T21:36:34,501 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting c575e90fea5346f4ad41769b459cb5b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732743392606 2024-11-27T21:36:34,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-27T21:36:34,507 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b39500f7a69456fd382cec5091ca715b#B#compaction#424 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:34,507 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/260d3499d54f424eabb572b7ca3a79dc is 50, key is test_row_0/B:col10/1732743392606/Put/seqid=0 2024-11-27T21:36:34,511 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b39500f7a69456fd382cec5091ca715b#A#compaction#425 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:34,512 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/b6eb3ab0845a4209930b74e1057a420c is 50, key is test_row_0/A:col10/1732743392606/Put/seqid=0 2024-11-27T21:36:34,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742323_1499 (size=12459) 2024-11-27T21:36:34,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742324_1500 (size=12459) 2024-11-27T21:36:34,520 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/b6eb3ab0845a4209930b74e1057a420c as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/b6eb3ab0845a4209930b74e1057a420c 2024-11-27T21:36:34,524 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b39500f7a69456fd382cec5091ca715b/A of b39500f7a69456fd382cec5091ca715b into b6eb3ab0845a4209930b74e1057a420c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:34,524 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:34,524 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., storeName=b39500f7a69456fd382cec5091ca715b/A, priority=13, startTime=1732743394499; duration=0sec 2024-11-27T21:36:34,524 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:36:34,524 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b39500f7a69456fd382cec5091ca715b:A 2024-11-27T21:36:34,524 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:34,526 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:34,526 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): b39500f7a69456fd382cec5091ca715b/C is initiating minor compaction (all files) 2024-11-27T21:36:34,526 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b39500f7a69456fd382cec5091ca715b/C in TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:34,526 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/45e60194b9074f328f6a5d2ebcb9057f, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/77981dbfb44f42a0b728938f5d5e2774, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/2e663844ed264e3993c9ad55a44d35c3] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp, totalSize=35.8 K 2024-11-27T21:36:34,526 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45e60194b9074f328f6a5d2ebcb9057f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732743391751 2024-11-27T21:36:34,527 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77981dbfb44f42a0b728938f5d5e2774, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732743392445 2024-11-27T21:36:34,527 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e663844ed264e3993c9ad55a44d35c3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732743392606 2024-11-27T21:36:34,531 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b39500f7a69456fd382cec5091ca715b#C#compaction#426 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:34,532 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/578235b776494336b63216e1efc114bd is 50, key is test_row_0/C:col10/1732743392606/Put/seqid=0 2024-11-27T21:36:34,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742325_1501 (size=12459) 2024-11-27T21:36:34,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-27T21:36:34,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:34,641 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing b39500f7a69456fd382cec5091ca715b 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-27T21:36:34,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/ab52b5611c7a4d9a8fd9cdb4a2c529f1 is 50, key is test_row_0/A:col10/1732743393301/Put/seqid=0 2024-11-27T21:36:34,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742326_1502 (size=12151) 2024-11-27T21:36:34,918 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/260d3499d54f424eabb572b7ca3a79dc as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/260d3499d54f424eabb572b7ca3a79dc 2024-11-27T21:36:34,922 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b39500f7a69456fd382cec5091ca715b/B of b39500f7a69456fd382cec5091ca715b into 260d3499d54f424eabb572b7ca3a79dc(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:34,922 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:34,922 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., storeName=b39500f7a69456fd382cec5091ca715b/B, priority=13, startTime=1732743394499; duration=0sec 2024-11-27T21:36:34,922 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:34,922 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b39500f7a69456fd382cec5091ca715b:B 2024-11-27T21:36:34,940 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/578235b776494336b63216e1efc114bd as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/578235b776494336b63216e1efc114bd 2024-11-27T21:36:34,944 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b39500f7a69456fd382cec5091ca715b/C of b39500f7a69456fd382cec5091ca715b into 578235b776494336b63216e1efc114bd(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:34,944 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:34,944 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., storeName=b39500f7a69456fd382cec5091ca715b/C, priority=13, startTime=1732743394499; duration=0sec 2024-11-27T21:36:34,944 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:34,944 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b39500f7a69456fd382cec5091ca715b:C 2024-11-27T21:36:35,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-27T21:36:35,055 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/ab52b5611c7a4d9a8fd9cdb4a2c529f1 2024-11-27T21:36:35,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/049403e8621147a0907b873b31c4ca05 is 50, key is test_row_0/B:col10/1732743393301/Put/seqid=0 2024-11-27T21:36:35,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742327_1503 (size=12151) 2024-11-27T21:36:35,477 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/049403e8621147a0907b873b31c4ca05 2024-11-27T21:36:35,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on b39500f7a69456fd382cec5091ca715b 2024-11-27T21:36:35,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:35,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/bc5be490b9824866a5706a785773f936 is 50, key is test_row_0/C:col10/1732743393301/Put/seqid=0 2024-11-27T21:36:35,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742328_1504 (size=12151) 2024-11-27T21:36:35,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:35,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743455505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:35,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:35,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:35,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:35,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:35,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:35,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743455518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:35,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743455519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:35,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:35,532 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:35,532 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:35,532 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:35,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743455520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:35,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:35,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:35,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:35,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:35,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:35,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:35,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:35,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743455623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:35,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:35,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:35,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:35,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:35,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743455643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:35,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:35,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743455643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:35,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:35,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:35,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:35,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:35,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:35,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:35,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743455643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:35,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:35,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:35,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:35,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:35,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:35,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743455833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:35,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:35,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:35,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:35,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743455863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:35,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:35,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:35,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:35,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743455863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:35,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:35,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743455864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:35,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:35,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:35,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:35,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:35,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:35,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:35,889 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/bc5be490b9824866a5706a785773f936 2024-11-27T21:36:35,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/ab52b5611c7a4d9a8fd9cdb4a2c529f1 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/ab52b5611c7a4d9a8fd9cdb4a2c529f1 2024-11-27T21:36:35,900 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/ab52b5611c7a4d9a8fd9cdb4a2c529f1, entries=150, sequenceid=232, filesize=11.9 K 2024-11-27T21:36:35,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/049403e8621147a0907b873b31c4ca05 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/049403e8621147a0907b873b31c4ca05 2024-11-27T21:36:35,905 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/049403e8621147a0907b873b31c4ca05, entries=150, sequenceid=232, filesize=11.9 K 2024-11-27T21:36:35,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/bc5be490b9824866a5706a785773f936 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/bc5be490b9824866a5706a785773f936 2024-11-27T21:36:35,908 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/bc5be490b9824866a5706a785773f936, entries=150, sequenceid=232, filesize=11.9 K 2024-11-27T21:36:35,909 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for b39500f7a69456fd382cec5091ca715b in 1268ms, sequenceid=232, compaction requested=false 2024-11-27T21:36:35,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:35,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:35,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-27T21:36:35,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-27T21:36:35,915 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-27T21:36:35,915 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0350 sec 2024-11-27T21:36:35,922 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 2.0470 sec 2024-11-27T21:36:36,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-27T21:36:36,023 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:36:36,024 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:36:36,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-27T21:36:36,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-27T21:36:36,026 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:36:36,026 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:36:36,027 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:36:36,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-27T21:36:36,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on b39500f7a69456fd382cec5091ca715b 2024-11-27T21:36:36,156 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:36:36,156 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b39500f7a69456fd382cec5091ca715b 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-27T21:36:36,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/6c341149875e47a6bbd139257b025c56 is 50, key is test_row_0/A:col10/1732743396155/Put/seqid=0 2024-11-27T21:36:36,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-27T21:36:36,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:36,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:36,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:36,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:36,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:36,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:36,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742329_1505 (size=14541) 2024-11-27T21:36:36,181 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/6c341149875e47a6bbd139257b025c56 2024-11-27T21:36:36,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/e19d73b7a51047e4bf6dca9674dcb859 is 50, key is test_row_0/B:col10/1732743396155/Put/seqid=0 2024-11-27T21:36:36,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742330_1506 (size=12151) 2024-11-27T21:36:36,209 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/e19d73b7a51047e4bf6dca9674dcb859 2024-11-27T21:36:36,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/a681f9365014485cba32b31f252b272c is 50, key is test_row_0/C:col10/1732743396155/Put/seqid=0 2024-11-27T21:36:36,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:36,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743456209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:36,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:36,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:36,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:36,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743456211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:36,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:36,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:36,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743456212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:36,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:36,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:36,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:36,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:36,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:36,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:36,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:36,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743456217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:36,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:36,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:36,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:36,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742331_1507 (size=12151) 2024-11-27T21:36:36,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-27T21:36:36,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:36,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:36,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:36,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:36,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:36,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:36,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-27T21:36:36,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:36,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743456333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:36,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:36,346 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:36,346 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:36,346 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:36,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743456334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:36,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:36,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:36,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:36,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:36,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743456334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:36,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:36,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:36,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:36,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:36,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743456343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:36,352 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:36,352 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:36,352 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:36,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-27T21:36:36,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:36,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:36,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:36,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:36,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:36,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:36,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:36,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743456553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:36,559 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:36,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:36,559 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:36,559 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:36,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743456553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:36,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:36,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:36,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:36,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:36,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743456553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:36,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:36,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:36,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:36,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:36,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743456564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:36,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:36,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:36,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:36,635 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/a681f9365014485cba32b31f252b272c 2024-11-27T21:36:36,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-27T21:36:36,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:36,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:36,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:36,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:36,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:36,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:36,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/6c341149875e47a6bbd139257b025c56 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/6c341149875e47a6bbd139257b025c56 2024-11-27T21:36:36,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/6c341149875e47a6bbd139257b025c56, entries=200, sequenceid=253, filesize=14.2 K 2024-11-27T21:36:36,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/e19d73b7a51047e4bf6dca9674dcb859 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/e19d73b7a51047e4bf6dca9674dcb859 2024-11-27T21:36:36,653 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/e19d73b7a51047e4bf6dca9674dcb859, entries=150, sequenceid=253, filesize=11.9 K 2024-11-27T21:36:36,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-27T21:36:36,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/a681f9365014485cba32b31f252b272c as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/a681f9365014485cba32b31f252b272c 2024-11-27T21:36:36,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/a681f9365014485cba32b31f252b272c, entries=150, sequenceid=253, filesize=11.9 K 2024-11-27T21:36:36,668 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for b39500f7a69456fd382cec5091ca715b in 512ms, sequenceid=253, compaction requested=true 2024-11-27T21:36:36,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:36,669 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:36,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b39500f7a69456fd382cec5091ca715b:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:36:36,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:36,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b39500f7a69456fd382cec5091ca715b:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:36:36,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:36,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b39500f7a69456fd382cec5091ca715b:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:36:36,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:36:36,670 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:36,671 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39151 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:36,671 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): b39500f7a69456fd382cec5091ca715b/A is initiating minor compaction (all files) 2024-11-27T21:36:36,671 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b39500f7a69456fd382cec5091ca715b/A in TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:36,672 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/b6eb3ab0845a4209930b74e1057a420c, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/ab52b5611c7a4d9a8fd9cdb4a2c529f1, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/6c341149875e47a6bbd139257b025c56] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp, totalSize=38.2 K 2024-11-27T21:36:36,672 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:36,672 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6eb3ab0845a4209930b74e1057a420c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732743392606 2024-11-27T21:36:36,672 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): b39500f7a69456fd382cec5091ca715b/B is initiating minor compaction (all files) 2024-11-27T21:36:36,672 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b39500f7a69456fd382cec5091ca715b/B in TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:36,672 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/260d3499d54f424eabb572b7ca3a79dc, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/049403e8621147a0907b873b31c4ca05, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/e19d73b7a51047e4bf6dca9674dcb859] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp, totalSize=35.9 K 2024-11-27T21:36:36,672 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab52b5611c7a4d9a8fd9cdb4a2c529f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732743393271 2024-11-27T21:36:36,673 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c341149875e47a6bbd139257b025c56, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732743395504 2024-11-27T21:36:36,673 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 260d3499d54f424eabb572b7ca3a79dc, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732743392606 2024-11-27T21:36:36,673 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 049403e8621147a0907b873b31c4ca05, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732743393271 2024-11-27T21:36:36,673 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting e19d73b7a51047e4bf6dca9674dcb859, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732743395517 2024-11-27T21:36:36,682 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b39500f7a69456fd382cec5091ca715b#A#compaction#433 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:36,682 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b39500f7a69456fd382cec5091ca715b#B#compaction#434 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:36,682 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/00bad1011e84413192a3cb697f434bf7 is 50, key is test_row_0/A:col10/1732743396155/Put/seqid=0 2024-11-27T21:36:36,682 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/fe75974c517d4aaaa4ea0d6b01ec240d is 50, key is test_row_0/B:col10/1732743396155/Put/seqid=0 2024-11-27T21:36:36,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742332_1508 (size=12561) 2024-11-27T21:36:36,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742333_1509 (size=12561) 2024-11-27T21:36:36,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-27T21:36:36,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:36,794 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing b39500f7a69456fd382cec5091ca715b 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-27T21:36:36,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/1bc5ef6311974b4c9201b900c0b6f1ee is 50, key is test_row_0/A:col10/1732743396206/Put/seqid=0 2024-11-27T21:36:36,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742334_1510 (size=12301) 2024-11-27T21:36:36,823 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/1bc5ef6311974b4c9201b900c0b6f1ee 2024-11-27T21:36:36,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/402a13f446124c4daecdd2ad12999f0d is 50, key is test_row_0/B:col10/1732743396206/Put/seqid=0 2024-11-27T21:36:36,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742335_1511 (size=12301) 2024-11-27T21:36:36,858 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/402a13f446124c4daecdd2ad12999f0d 2024-11-27T21:36:36,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/227b1b1f04ce45aa8d1431594781b7c3 is 50, key is test_row_0/C:col10/1732743396206/Put/seqid=0 2024-11-27T21:36:36,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on b39500f7a69456fd382cec5091ca715b 2024-11-27T21:36:36,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:36,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742336_1512 (size=12301) 2024-11-27T21:36:36,902 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/227b1b1f04ce45aa8d1431594781b7c3 2024-11-27T21:36:36,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/1bc5ef6311974b4c9201b900c0b6f1ee as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/1bc5ef6311974b4c9201b900c0b6f1ee 2024-11-27T21:36:36,913 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/1bc5ef6311974b4c9201b900c0b6f1ee, entries=150, sequenceid=270, filesize=12.0 K 2024-11-27T21:36:36,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/402a13f446124c4daecdd2ad12999f0d as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/402a13f446124c4daecdd2ad12999f0d 2024-11-27T21:36:36,921 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/402a13f446124c4daecdd2ad12999f0d, entries=150, sequenceid=270, filesize=12.0 K 2024-11-27T21:36:36,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/227b1b1f04ce45aa8d1431594781b7c3 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/227b1b1f04ce45aa8d1431594781b7c3 2024-11-27T21:36:36,927 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/227b1b1f04ce45aa8d1431594781b7c3, entries=150, sequenceid=270, filesize=12.0 K 2024-11-27T21:36:36,928 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=87.22 KB/89310 for b39500f7a69456fd382cec5091ca715b in 134ms, sequenceid=270, compaction requested=true 2024-11-27T21:36:36,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:36,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:36,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-27T21:36:36,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-27T21:36:36,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-27T21:36:36,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 903 msec 2024-11-27T21:36:36,935 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 910 msec 2024-11-27T21:36:36,937 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:36:36,937 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b39500f7a69456fd382cec5091ca715b 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-27T21:36:36,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on b39500f7a69456fd382cec5091ca715b 2024-11-27T21:36:36,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/405c58fdd007435082c0ebd9b48a766e is 50, key is test_row_0/A:col10/1732743396924/Put/seqid=0 2024-11-27T21:36:36,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742337_1513 (size=17181) 2024-11-27T21:36:37,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:37,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743456996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:37,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:37,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:37,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743456997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:37,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:37,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743456998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:37,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:37,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:37,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:37,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743457010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:37,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:37,099 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/fe75974c517d4aaaa4ea0d6b01ec240d as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/fe75974c517d4aaaa4ea0d6b01ec240d 2024-11-27T21:36:37,101 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/00bad1011e84413192a3cb697f434bf7 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/00bad1011e84413192a3cb697f434bf7 2024-11-27T21:36:37,104 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b39500f7a69456fd382cec5091ca715b/B of b39500f7a69456fd382cec5091ca715b into fe75974c517d4aaaa4ea0d6b01ec240d(size=12.3 K), total size for store is 24.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:37,104 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:37,104 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., storeName=b39500f7a69456fd382cec5091ca715b/B, priority=13, startTime=1732743396669; duration=0sec 2024-11-27T21:36:37,105 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:36:37,105 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b39500f7a69456fd382cec5091ca715b:B 2024-11-27T21:36:37,105 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:36:37,108 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49062 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:36:37,108 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b39500f7a69456fd382cec5091ca715b/A of b39500f7a69456fd382cec5091ca715b into 00bad1011e84413192a3cb697f434bf7(size=12.3 K), total size for store is 24.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:37,108 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:37,108 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): b39500f7a69456fd382cec5091ca715b/C is initiating minor compaction (all files) 2024-11-27T21:36:37,108 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., storeName=b39500f7a69456fd382cec5091ca715b/A, priority=13, startTime=1732743396669; duration=0sec 2024-11-27T21:36:37,108 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b39500f7a69456fd382cec5091ca715b/C in TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:37,108 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/578235b776494336b63216e1efc114bd, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/bc5be490b9824866a5706a785773f936, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/a681f9365014485cba32b31f252b272c, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/227b1b1f04ce45aa8d1431594781b7c3] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp, totalSize=47.9 K 2024-11-27T21:36:37,109 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:37,109 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b39500f7a69456fd382cec5091ca715b:A 2024-11-27T21:36:37,109 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 578235b776494336b63216e1efc114bd, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732743392606 2024-11-27T21:36:37,115 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting bc5be490b9824866a5706a785773f936, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732743393271 2024-11-27T21:36:37,115 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting a681f9365014485cba32b31f252b272c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732743395517 2024-11-27T21:36:37,115 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 227b1b1f04ce45aa8d1431594781b7c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732743396206 2024-11-27T21:36:37,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:37,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743457114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:37,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:37,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:37,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743457123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:37,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:37,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743457123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:37,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:37,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:37,135 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b39500f7a69456fd382cec5091ca715b#C#compaction#439 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:37,136 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/f35e78358c364ce48a61ff061697457e is 50, key is test_row_0/C:col10/1732743396206/Put/seqid=0 2024-11-27T21:36:37,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:37,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743457143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:37,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:37,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-27T21:36:37,164 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:36:37,165 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:36:37,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-27T21:36:37,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-27T21:36:37,167 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:36:37,167 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:36:37,167 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:36:37,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742338_1514 (size=12745) 2024-11-27T21:36:37,199 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/f35e78358c364ce48a61ff061697457e as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/f35e78358c364ce48a61ff061697457e 2024-11-27T21:36:37,204 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b39500f7a69456fd382cec5091ca715b/C of b39500f7a69456fd382cec5091ca715b into f35e78358c364ce48a61ff061697457e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:37,204 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:37,204 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., storeName=b39500f7a69456fd382cec5091ca715b/C, priority=12, startTime=1732743396669; duration=0sec 2024-11-27T21:36:37,204 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:37,204 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b39500f7a69456fd382cec5091ca715b:C 2024-11-27T21:36:37,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-27T21:36:37,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-27T21:36:37,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:37,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:37,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:37,321 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:37,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:37,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:37,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:37,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743457333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:37,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:37,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:37,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743457343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:37,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:37,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743457343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:37,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:37,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:37,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:37,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743457354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:37,362 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,362 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,362 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:37,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/405c58fdd007435082c0ebd9b48a766e 2024-11-27T21:36:37,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/f51a8690ae3e4033b699eb675f53f2ec is 50, key is test_row_0/B:col10/1732743396924/Put/seqid=0 2024-11-27T21:36:37,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742339_1515 (size=12301) 2024-11-27T21:36:37,424 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/f51a8690ae3e4033b699eb675f53f2ec 2024-11-27T21:36:37,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/5ff8b26f8c1e4c2a8b3224c2cd31abc9 is 50, key is test_row_0/C:col10/1732743396924/Put/seqid=0 2024-11-27T21:36:37,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742340_1516 (size=12301) 2024-11-27T21:36:37,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/5ff8b26f8c1e4c2a8b3224c2cd31abc9 2024-11-27T21:36:37,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/405c58fdd007435082c0ebd9b48a766e as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/405c58fdd007435082c0ebd9b48a766e 2024-11-27T21:36:37,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-27T21:36:37,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:37,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:37,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:37,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:37,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:37,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:37,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/405c58fdd007435082c0ebd9b48a766e, entries=250, sequenceid=288, filesize=16.8 K 2024-11-27T21:36:37,477 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-27T21:36:37,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/f51a8690ae3e4033b699eb675f53f2ec as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/f51a8690ae3e4033b699eb675f53f2ec 2024-11-27T21:36:37,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-27T21:36:37,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/f51a8690ae3e4033b699eb675f53f2ec, entries=150, sequenceid=288, filesize=12.0 K 2024-11-27T21:36:37,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/5ff8b26f8c1e4c2a8b3224c2cd31abc9 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/5ff8b26f8c1e4c2a8b3224c2cd31abc9 2024-11-27T21:36:37,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/5ff8b26f8c1e4c2a8b3224c2cd31abc9, entries=150, sequenceid=288, filesize=12.0 K 2024-11-27T21:36:37,496 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for b39500f7a69456fd382cec5091ca715b in 559ms, sequenceid=288, compaction requested=true 2024-11-27T21:36:37,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:37,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b39500f7a69456fd382cec5091ca715b:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:36:37,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:37,496 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:37,496 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:37,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b39500f7a69456fd382cec5091ca715b:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:36:37,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:37,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b39500f7a69456fd382cec5091ca715b:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:36:37,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:36:37,497 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42043 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:37,497 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37163 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:37,497 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): b39500f7a69456fd382cec5091ca715b/A is initiating minor compaction (all files) 2024-11-27T21:36:37,497 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): b39500f7a69456fd382cec5091ca715b/B is initiating minor compaction (all files) 2024-11-27T21:36:37,497 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b39500f7a69456fd382cec5091ca715b/A in TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:37,497 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b39500f7a69456fd382cec5091ca715b/B in TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:37,497 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/00bad1011e84413192a3cb697f434bf7, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/1bc5ef6311974b4c9201b900c0b6f1ee, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/405c58fdd007435082c0ebd9b48a766e] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp, totalSize=41.1 K 2024-11-27T21:36:37,497 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/fe75974c517d4aaaa4ea0d6b01ec240d, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/402a13f446124c4daecdd2ad12999f0d, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/f51a8690ae3e4033b699eb675f53f2ec] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp, totalSize=36.3 K 2024-11-27T21:36:37,498 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00bad1011e84413192a3cb697f434bf7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732743395517 2024-11-27T21:36:37,498 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting fe75974c517d4aaaa4ea0d6b01ec240d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732743395517 2024-11-27T21:36:37,498 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1bc5ef6311974b4c9201b900c0b6f1ee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732743396206 2024-11-27T21:36:37,498 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 402a13f446124c4daecdd2ad12999f0d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732743396206 2024-11-27T21:36:37,498 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 405c58fdd007435082c0ebd9b48a766e, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732743396903 2024-11-27T21:36:37,498 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting f51a8690ae3e4033b699eb675f53f2ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732743396903 2024-11-27T21:36:37,506 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b39500f7a69456fd382cec5091ca715b#B#compaction#442 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:37,506 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/86318238ea354f5fab4ad744b914e8d2 is 50, key is test_row_0/B:col10/1732743396924/Put/seqid=0 2024-11-27T21:36:37,506 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b39500f7a69456fd382cec5091ca715b#A#compaction#443 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:37,507 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/efad840aba294fd493300574b5b78f74 is 50, key is test_row_0/A:col10/1732743396924/Put/seqid=0 2024-11-27T21:36:37,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742342_1518 (size=12813) 2024-11-27T21:36:37,552 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/efad840aba294fd493300574b5b78f74 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/efad840aba294fd493300574b5b78f74 2024-11-27T21:36:37,560 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b39500f7a69456fd382cec5091ca715b/A of b39500f7a69456fd382cec5091ca715b into efad840aba294fd493300574b5b78f74(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:37,560 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:37,560 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., storeName=b39500f7a69456fd382cec5091ca715b/A, priority=13, startTime=1732743397496; duration=0sec 2024-11-27T21:36:37,560 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:36:37,560 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b39500f7a69456fd382cec5091ca715b:A 2024-11-27T21:36:37,560 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-27T21:36:37,561 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-27T21:36:37,561 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-27T21:36:37,561 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. because compaction request was cancelled 2024-11-27T21:36:37,561 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b39500f7a69456fd382cec5091ca715b:C 2024-11-27T21:36:37,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742341_1517 (size=12813) 2024-11-27T21:36:37,570 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/86318238ea354f5fab4ad744b914e8d2 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/86318238ea354f5fab4ad744b914e8d2 2024-11-27T21:36:37,576 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b39500f7a69456fd382cec5091ca715b/B of b39500f7a69456fd382cec5091ca715b into 86318238ea354f5fab4ad744b914e8d2(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:37,576 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:37,576 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., storeName=b39500f7a69456fd382cec5091ca715b/B, priority=13, startTime=1732743397496; duration=0sec 2024-11-27T21:36:37,577 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:37,577 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b39500f7a69456fd382cec5091ca715b:B 2024-11-27T21:36:37,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-27T21:36:37,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:37,627 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing b39500f7a69456fd382cec5091ca715b 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-27T21:36:37,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/2b8618b677a74c98a10582413bcb7fd2 is 50, key is test_row_0/A:col10/1732743396996/Put/seqid=0 2024-11-27T21:36:37,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742343_1519 (size=12301) 2024-11-27T21:36:37,656 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/2b8618b677a74c98a10582413bcb7fd2 2024-11-27T21:36:37,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on b39500f7a69456fd382cec5091ca715b 2024-11-27T21:36:37,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:37,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/c108ec03c0c2479eaa74110d850e73bd is 50, key is test_row_0/B:col10/1732743396996/Put/seqid=0 2024-11-27T21:36:37,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742344_1520 (size=12301) 2024-11-27T21:36:37,676 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/c108ec03c0c2479eaa74110d850e73bd 2024-11-27T21:36:37,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/d05aafdc1b694d738b9bcde5bb13f109 is 50, key is test_row_0/C:col10/1732743396996/Put/seqid=0 2024-11-27T21:36:37,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:37,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742345_1521 (size=12301) 2024-11-27T21:36:37,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743457721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:37,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:37,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743457722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:37,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:37,736 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/d05aafdc1b694d738b9bcde5bb13f109 2024-11-27T21:36:37,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:37,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:37,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743457723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:37,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:37,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743457724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:37,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:37,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:37,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/2b8618b677a74c98a10582413bcb7fd2 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/2b8618b677a74c98a10582413bcb7fd2 2024-11-27T21:36:37,746 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/2b8618b677a74c98a10582413bcb7fd2, entries=150, sequenceid=311, filesize=12.0 K 2024-11-27T21:36:37,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/c108ec03c0c2479eaa74110d850e73bd as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/c108ec03c0c2479eaa74110d850e73bd 2024-11-27T21:36:37,761 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/c108ec03c0c2479eaa74110d850e73bd, entries=150, sequenceid=311, filesize=12.0 K 2024-11-27T21:36:37,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/d05aafdc1b694d738b9bcde5bb13f109 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/d05aafdc1b694d738b9bcde5bb13f109 2024-11-27T21:36:37,767 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/d05aafdc1b694d738b9bcde5bb13f109, entries=150, sequenceid=311, filesize=12.0 K 2024-11-27T21:36:37,768 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for b39500f7a69456fd382cec5091ca715b in 141ms, sequenceid=311, compaction requested=true 2024-11-27T21:36:37,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:37,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:37,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-27T21:36:37,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-27T21:36:37,772 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-27T21:36:37,772 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 603 msec 2024-11-27T21:36:37,774 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 607 msec 2024-11-27T21:36:37,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-27T21:36:37,795 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:36:37,796 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:36:37,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-27T21:36:37,798 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:36:37,799 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:36:37,799 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:36:37,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-27T21:36:37,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on b39500f7a69456fd382cec5091ca715b 2024-11-27T21:36:37,813 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:36:37,813 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b39500f7a69456fd382cec5091ca715b 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-27T21:36:37,818 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/89898fd9930a4997bcadda89260ea3bd is 50, key is test_row_0/A:col10/1732743397706/Put/seqid=0 2024-11-27T21:36:37,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742346_1522 (size=14741) 2024-11-27T21:36:37,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:37,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743457861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:37,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:37,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:37,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743457862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:37,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:37,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743457863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:37,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:37,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743457864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:37,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:37,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:37,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:37,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1732743457867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:37,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:37,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:37,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-27T21:36:37,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-27T21:36:37,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:37,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:37,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:37,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:37,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:37,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:37,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:37,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743457973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:37,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:37,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:37,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743457983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:37,989 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:37,989 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,989 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:37,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743457983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:37,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:37,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743457983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:37,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:37,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:37,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1732743457984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:37,990 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,990 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,990 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:37,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:37,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:38,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-27T21:36:38,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:38,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:38,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:38,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:38,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:38,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:38,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-27T21:36:38,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:38,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743458193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:38,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:38,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:38,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:38,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743458194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:38,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:38,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:38,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743458194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:38,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:38,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:38,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:38,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:38,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1732743458194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:38,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:38,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:38,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:38,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:38,200 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:38,200 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:38,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743458194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:38,200 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:38,201 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:38,201 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:38,201 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:38,227 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/89898fd9930a4997bcadda89260ea3bd 2024-11-27T21:36:38,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/081123bcbd4b41f8b783856faae553cb is 50, key is test_row_0/B:col10/1732743397706/Put/seqid=0 2024-11-27T21:36:38,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742347_1523 (size=12301) 2024-11-27T21:36:38,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/081123bcbd4b41f8b783856faae553cb 2024-11-27T21:36:38,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/b79b9965e40240a586fe0017c4faa4be is 50, key is test_row_0/C:col10/1732743397706/Put/seqid=0 2024-11-27T21:36:38,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-27T21:36:38,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:38,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:38,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:38,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:38,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:38,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:38,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742348_1524 (size=12301) 2024-11-27T21:36:38,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-27T21:36:38,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:38,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:38,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:38,413 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:38,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:38,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:38,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-27T21:36:38,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:38,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743458503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:38,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:38,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:38,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:38,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:38,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1732743458503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:38,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:38,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:38,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:38,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:38,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743458503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:38,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:38,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:38,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:38,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:38,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743458503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:38,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:38,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:38,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:38,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:38,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743458503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:38,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:38,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:38,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:38,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-27T21:36:38,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:38,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:38,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:38,567 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:38,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:38,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:38,665 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/b79b9965e40240a586fe0017c4faa4be 2024-11-27T21:36:38,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/89898fd9930a4997bcadda89260ea3bd as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/89898fd9930a4997bcadda89260ea3bd 2024-11-27T21:36:38,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/89898fd9930a4997bcadda89260ea3bd, entries=200, sequenceid=330, filesize=14.4 K 2024-11-27T21:36:38,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/081123bcbd4b41f8b783856faae553cb as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/081123bcbd4b41f8b783856faae553cb 2024-11-27T21:36:38,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/081123bcbd4b41f8b783856faae553cb, entries=150, sequenceid=330, filesize=12.0 K 2024-11-27T21:36:38,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/b79b9965e40240a586fe0017c4faa4be as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/b79b9965e40240a586fe0017c4faa4be 2024-11-27T21:36:38,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/b79b9965e40240a586fe0017c4faa4be, entries=150, sequenceid=330, filesize=12.0 K 2024-11-27T21:36:38,689 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for b39500f7a69456fd382cec5091ca715b in 876ms, sequenceid=330, compaction requested=true 2024-11-27T21:36:38,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:38,689 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:38,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b39500f7a69456fd382cec5091ca715b:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:36:38,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:38,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b39500f7a69456fd382cec5091ca715b:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:36:38,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:36:38,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b39500f7a69456fd382cec5091ca715b:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:36:38,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:36:38,690 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:38,690 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39855 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:38,690 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): b39500f7a69456fd382cec5091ca715b/A is initiating minor compaction (all files) 2024-11-27T21:36:38,690 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b39500f7a69456fd382cec5091ca715b/A in TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:38,690 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/efad840aba294fd493300574b5b78f74, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/2b8618b677a74c98a10582413bcb7fd2, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/89898fd9930a4997bcadda89260ea3bd] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp, totalSize=38.9 K 2024-11-27T21:36:38,693 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:38,693 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting efad840aba294fd493300574b5b78f74, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732743396903 2024-11-27T21:36:38,693 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): b39500f7a69456fd382cec5091ca715b/B is initiating minor compaction (all files) 2024-11-27T21:36:38,693 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b39500f7a69456fd382cec5091ca715b/B in TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:38,693 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/86318238ea354f5fab4ad744b914e8d2, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/c108ec03c0c2479eaa74110d850e73bd, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/081123bcbd4b41f8b783856faae553cb] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp, totalSize=36.5 K 2024-11-27T21:36:38,693 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b8618b677a74c98a10582413bcb7fd2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732743396994 2024-11-27T21:36:38,693 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 86318238ea354f5fab4ad744b914e8d2, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732743396903 2024-11-27T21:36:38,693 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting c108ec03c0c2479eaa74110d850e73bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732743396994 2024-11-27T21:36:38,694 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89898fd9930a4997bcadda89260ea3bd, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732743397706 2024-11-27T21:36:38,694 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 081123bcbd4b41f8b783856faae553cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732743397706 2024-11-27T21:36:38,703 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b39500f7a69456fd382cec5091ca715b#A#compaction#450 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:38,703 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/280cdffa3760460c9121b1b67b7cd007 is 50, key is test_row_0/A:col10/1732743397706/Put/seqid=0 2024-11-27T21:36:38,705 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b39500f7a69456fd382cec5091ca715b#B#compaction#451 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:38,706 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/467b8e85727245c4ae7744a07d8fbeb8 is 50, key is test_row_0/B:col10/1732743397706/Put/seqid=0 2024-11-27T21:36:38,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-27T21:36:38,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:38,720 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing b39500f7a69456fd382cec5091ca715b 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-27T21:36:38,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742350_1526 (size=12915) 2024-11-27T21:36:38,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742349_1525 (size=12915) 2024-11-27T21:36:38,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/3c48bdc98c0542aa86533148520c196f is 50, key is test_row_0/A:col10/1732743397866/Put/seqid=0 2024-11-27T21:36:38,736 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/467b8e85727245c4ae7744a07d8fbeb8 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/467b8e85727245c4ae7744a07d8fbeb8 2024-11-27T21:36:38,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742351_1527 (size=12301) 2024-11-27T21:36:38,740 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b39500f7a69456fd382cec5091ca715b/B of b39500f7a69456fd382cec5091ca715b into 467b8e85727245c4ae7744a07d8fbeb8(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:38,740 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:38,740 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., storeName=b39500f7a69456fd382cec5091ca715b/B, priority=13, startTime=1732743398690; duration=0sec 2024-11-27T21:36:38,740 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:36:38,740 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b39500f7a69456fd382cec5091ca715b:B 2024-11-27T21:36:38,741 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:36:38,741 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/3c48bdc98c0542aa86533148520c196f 2024-11-27T21:36:38,742 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49648 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:36:38,742 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): b39500f7a69456fd382cec5091ca715b/C is initiating minor compaction (all files) 2024-11-27T21:36:38,742 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b39500f7a69456fd382cec5091ca715b/C in TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:38,742 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/f35e78358c364ce48a61ff061697457e, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/5ff8b26f8c1e4c2a8b3224c2cd31abc9, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/d05aafdc1b694d738b9bcde5bb13f109, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/b79b9965e40240a586fe0017c4faa4be] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp, totalSize=48.5 K 2024-11-27T21:36:38,743 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting f35e78358c364ce48a61ff061697457e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732743396206 2024-11-27T21:36:38,743 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ff8b26f8c1e4c2a8b3224c2cd31abc9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732743396903 2024-11-27T21:36:38,743 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting d05aafdc1b694d738b9bcde5bb13f109, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732743396994 2024-11-27T21:36:38,743 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting b79b9965e40240a586fe0017c4faa4be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732743397706 2024-11-27T21:36:38,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/9d33aad173344bed8dc394388f02cb00 is 50, key is test_row_0/B:col10/1732743397866/Put/seqid=0 2024-11-27T21:36:38,767 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b39500f7a69456fd382cec5091ca715b#C#compaction#454 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:38,767 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/2afc13f6865c45649b61e6995219dfad is 50, key is test_row_0/C:col10/1732743397706/Put/seqid=0 2024-11-27T21:36:38,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742352_1528 (size=12301) 2024-11-27T21:36:38,783 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/9d33aad173344bed8dc394388f02cb00 2024-11-27T21:36:38,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742353_1529 (size=12881) 2024-11-27T21:36:38,793 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/2afc13f6865c45649b61e6995219dfad as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/2afc13f6865c45649b61e6995219dfad 2024-11-27T21:36:38,804 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b39500f7a69456fd382cec5091ca715b/C of b39500f7a69456fd382cec5091ca715b into 2afc13f6865c45649b61e6995219dfad(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:38,805 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:38,805 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., storeName=b39500f7a69456fd382cec5091ca715b/C, priority=12, startTime=1732743398690; duration=0sec 2024-11-27T21:36:38,805 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:38,805 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b39500f7a69456fd382cec5091ca715b:C 2024-11-27T21:36:38,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/027b6a664210432391bd31eac56c8657 is 50, key is test_row_0/C:col10/1732743397866/Put/seqid=0 2024-11-27T21:36:38,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742354_1530 (size=12301) 2024-11-27T21:36:38,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-27T21:36:39,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on b39500f7a69456fd382cec5091ca715b 2024-11-27T21:36:39,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:39,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:39,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743459061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:39,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:39,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:39,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743459061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:39,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:39,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:39,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743459062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:39,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:39,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1732743459062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:39,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:39,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:39,075 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,075 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,075 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:39,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743459073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:39,075 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,075 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,075 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:39,130 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/280cdffa3760460c9121b1b67b7cd007 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/280cdffa3760460c9121b1b67b7cd007 2024-11-27T21:36:39,136 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b39500f7a69456fd382cec5091ca715b/A of b39500f7a69456fd382cec5091ca715b into 280cdffa3760460c9121b1b67b7cd007(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:39,136 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:39,136 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., storeName=b39500f7a69456fd382cec5091ca715b/A, priority=13, startTime=1732743398689; duration=0sec 2024-11-27T21:36:39,136 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:39,136 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b39500f7a69456fd382cec5091ca715b:A 2024-11-27T21:36:39,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:39,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1732743459183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:39,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:39,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743459183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:39,189 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,189 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,189 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:39,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:39,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743459183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:39,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:39,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:39,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:39,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743459184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:39,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:39,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:39,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743459184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:39,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:39,222 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/027b6a664210432391bd31eac56c8657 2024-11-27T21:36:39,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/3c48bdc98c0542aa86533148520c196f as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/3c48bdc98c0542aa86533148520c196f 2024-11-27T21:36:39,237 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/3c48bdc98c0542aa86533148520c196f, entries=150, sequenceid=347, filesize=12.0 K 2024-11-27T21:36:39,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/9d33aad173344bed8dc394388f02cb00 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/9d33aad173344bed8dc394388f02cb00 2024-11-27T21:36:39,241 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/9d33aad173344bed8dc394388f02cb00, entries=150, sequenceid=347, filesize=12.0 K 2024-11-27T21:36:39,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/027b6a664210432391bd31eac56c8657 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/027b6a664210432391bd31eac56c8657 2024-11-27T21:36:39,244 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/027b6a664210432391bd31eac56c8657, entries=150, sequenceid=347, filesize=12.0 K 2024-11-27T21:36:39,245 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for b39500f7a69456fd382cec5091ca715b in 525ms, sequenceid=347, compaction requested=false 2024-11-27T21:36:39,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:39,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:39,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-27T21:36:39,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-27T21:36:39,248 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-27T21:36:39,248 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4470 sec 2024-11-27T21:36:39,250 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 1.4520 sec 2024-11-27T21:36:39,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on b39500f7a69456fd382cec5091ca715b 2024-11-27T21:36:39,399 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:36:39,399 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b39500f7a69456fd382cec5091ca715b 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-27T21:36:39,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/b913858e7f404dc3b0a632e552268e68 is 50, key is test_row_0/A:col10/1732743399059/Put/seqid=0 2024-11-27T21:36:39,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742355_1531 (size=14741) 2024-11-27T21:36:39,418 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/b913858e7f404dc3b0a632e552268e68 2024-11-27T21:36:39,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/2fd5cfeeecfb41f4a400cf7eae01a466 is 50, key is test_row_0/B:col10/1732743399059/Put/seqid=0 2024-11-27T21:36:39,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742356_1532 (size=12301) 2024-11-27T21:36:39,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:39,434 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/2fd5cfeeecfb41f4a400cf7eae01a466 2024-11-27T21:36:39,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743459417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:39,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:39,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:39,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1732743459427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:39,439 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,439 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,439 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:39,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/7a02a2bbe66a43c5a7c25db1bcd68eab is 50, key is test_row_0/C:col10/1732743399059/Put/seqid=0 2024-11-27T21:36:39,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742357_1533 (size=12301) 2024-11-27T21:36:39,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:39,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743459432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:39,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:39,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743459433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:39,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:39,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:39,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743459434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:39,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:39,450 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,450 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,450 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:39,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:39,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743459543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:39,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:39,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1732743459544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:39,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:39,548 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,548 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,548 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:39,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:39,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743459553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:39,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:39,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743459553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:39,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:39,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:39,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743459553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:39,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:39,562 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,562 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,562 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:39,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:39,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743459754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:39,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:39,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:39,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1732743459754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:39,761 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,761 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,761 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:39,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:39,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743459763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:39,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:39,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:39,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743459763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:39,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:39,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:39,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743459773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:39,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:39,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:39,845 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/7a02a2bbe66a43c5a7c25db1bcd68eab 2024-11-27T21:36:39,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/b913858e7f404dc3b0a632e552268e68 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/b913858e7f404dc3b0a632e552268e68 2024-11-27T21:36:39,854 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/b913858e7f404dc3b0a632e552268e68, entries=200, sequenceid=370, filesize=14.4 K 2024-11-27T21:36:39,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/2fd5cfeeecfb41f4a400cf7eae01a466 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/2fd5cfeeecfb41f4a400cf7eae01a466 2024-11-27T21:36:39,857 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/2fd5cfeeecfb41f4a400cf7eae01a466, entries=150, sequenceid=370, filesize=12.0 K 2024-11-27T21:36:39,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/7a02a2bbe66a43c5a7c25db1bcd68eab as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/7a02a2bbe66a43c5a7c25db1bcd68eab 2024-11-27T21:36:39,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/7a02a2bbe66a43c5a7c25db1bcd68eab, entries=150, sequenceid=370, filesize=12.0 K 2024-11-27T21:36:39,862 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for b39500f7a69456fd382cec5091ca715b in 463ms, sequenceid=370, compaction requested=true 2024-11-27T21:36:39,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:39,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b39500f7a69456fd382cec5091ca715b:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:36:39,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:39,862 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:39,863 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:39,863 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39957 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:39,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b39500f7a69456fd382cec5091ca715b:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:36:39,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:39,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b39500f7a69456fd382cec5091ca715b:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:36:39,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:36:39,863 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): b39500f7a69456fd382cec5091ca715b/A is initiating minor compaction (all files) 2024-11-27T21:36:39,863 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b39500f7a69456fd382cec5091ca715b/A in TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:39,863 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/280cdffa3760460c9121b1b67b7cd007, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/3c48bdc98c0542aa86533148520c196f, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/b913858e7f404dc3b0a632e552268e68] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp, totalSize=39.0 K 2024-11-27T21:36:39,864 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:39,864 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 280cdffa3760460c9121b1b67b7cd007, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732743397706 2024-11-27T21:36:39,864 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): b39500f7a69456fd382cec5091ca715b/B is initiating minor compaction (all files) 2024-11-27T21:36:39,864 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c48bdc98c0542aa86533148520c196f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732743397855 2024-11-27T21:36:39,864 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b39500f7a69456fd382cec5091ca715b/B in TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:39,864 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/467b8e85727245c4ae7744a07d8fbeb8, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/9d33aad173344bed8dc394388f02cb00, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/2fd5cfeeecfb41f4a400cf7eae01a466] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp, totalSize=36.6 K 2024-11-27T21:36:39,864 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting b913858e7f404dc3b0a632e552268e68, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732743399059 2024-11-27T21:36:39,865 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 467b8e85727245c4ae7744a07d8fbeb8, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732743397706 2024-11-27T21:36:39,865 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d33aad173344bed8dc394388f02cb00, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732743397855 2024-11-27T21:36:39,865 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fd5cfeeecfb41f4a400cf7eae01a466, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732743399059 2024-11-27T21:36:39,874 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b39500f7a69456fd382cec5091ca715b#A#compaction#459 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:39,875 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/75a6c69049b1418fb9598d761bdee04d is 50, key is test_row_0/A:col10/1732743399059/Put/seqid=0 2024-11-27T21:36:39,880 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b39500f7a69456fd382cec5091ca715b#B#compaction#460 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:39,880 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/f773ba42607c4dbba9315848bbf1a8f6 is 50, key is test_row_0/B:col10/1732743399059/Put/seqid=0 2024-11-27T21:36:39,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742359_1535 (size=13017) 2024-11-27T21:36:39,908 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/f773ba42607c4dbba9315848bbf1a8f6 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/f773ba42607c4dbba9315848bbf1a8f6 2024-11-27T21:36:39,913 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b39500f7a69456fd382cec5091ca715b/B of b39500f7a69456fd382cec5091ca715b into f773ba42607c4dbba9315848bbf1a8f6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:39,913 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:39,913 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., storeName=b39500f7a69456fd382cec5091ca715b/B, priority=13, startTime=1732743399863; duration=0sec 2024-11-27T21:36:39,913 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:36:39,913 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b39500f7a69456fd382cec5091ca715b:B 2024-11-27T21:36:39,913 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:39,914 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:39,914 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): b39500f7a69456fd382cec5091ca715b/C is initiating minor compaction (all files) 2024-11-27T21:36:39,914 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b39500f7a69456fd382cec5091ca715b/C in TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:39,914 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/2afc13f6865c45649b61e6995219dfad, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/027b6a664210432391bd31eac56c8657, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/7a02a2bbe66a43c5a7c25db1bcd68eab] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp, totalSize=36.6 K 2024-11-27T21:36:39,914 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 2afc13f6865c45649b61e6995219dfad, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732743397706 2024-11-27T21:36:39,914 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 027b6a664210432391bd31eac56c8657, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732743397855 2024-11-27T21:36:39,915 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a02a2bbe66a43c5a7c25db1bcd68eab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732743399059 2024-11-27T21:36:39,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742358_1534 (size=13017) 2024-11-27T21:36:39,929 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b39500f7a69456fd382cec5091ca715b#C#compaction#461 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:39,930 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/0b9202d8eeb54541a6d275ef823bf366 is 50, key is test_row_0/C:col10/1732743399059/Put/seqid=0 2024-11-27T21:36:39,930 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/75a6c69049b1418fb9598d761bdee04d as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/75a6c69049b1418fb9598d761bdee04d 2024-11-27T21:36:39,936 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b39500f7a69456fd382cec5091ca715b/A of b39500f7a69456fd382cec5091ca715b into 75a6c69049b1418fb9598d761bdee04d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:39,937 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:39,937 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., storeName=b39500f7a69456fd382cec5091ca715b/A, priority=13, startTime=1732743399862; duration=0sec 2024-11-27T21:36:39,937 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:39,937 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b39500f7a69456fd382cec5091ca715b:A 2024-11-27T21:36:39,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-27T21:36:39,944 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:36:39,944 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:36:39,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees 2024-11-27T21:36:39,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-27T21:36:39,946 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:36:39,947 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:36:39,947 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:36:39,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742360_1536 (size=12983) 2024-11-27T21:36:39,954 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/0b9202d8eeb54541a6d275ef823bf366 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/0b9202d8eeb54541a6d275ef823bf366 2024-11-27T21:36:39,962 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b39500f7a69456fd382cec5091ca715b/C of b39500f7a69456fd382cec5091ca715b into 0b9202d8eeb54541a6d275ef823bf366(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:39,962 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:39,962 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., storeName=b39500f7a69456fd382cec5091ca715b/C, priority=13, startTime=1732743399863; duration=0sec 2024-11-27T21:36:39,962 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:39,962 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b39500f7a69456fd382cec5091ca715b:C 2024-11-27T21:36:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-27T21:36:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on b39500f7a69456fd382cec5091ca715b 2024-11-27T21:36:40,078 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:36:40,078 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b39500f7a69456fd382cec5091ca715b 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-27T21:36:40,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/b9ce04f9e1f14334b3eb95f0a29fc800 is 50, key is test_row_0/A:col10/1732743400077/Put/seqid=0 2024-11-27T21:36:40,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-27T21:36:40,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742361_1537 (size=14741) 2024-11-27T21:36:40,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:40,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:40,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:40,099 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:40,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:40,100 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/b9ce04f9e1f14334b3eb95f0a29fc800 2024-11-27T21:36:40,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:40,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/63342e8be8114ad2baa453ad7f14d7ed is 50, key is test_row_0/B:col10/1732743400077/Put/seqid=0 2024-11-27T21:36:40,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742362_1538 (size=12301) 2024-11-27T21:36:40,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/63342e8be8114ad2baa453ad7f14d7ed 2024-11-27T21:36:40,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/cf0f53912c5f487db482ae33bb30da53 is 50, key is test_row_0/C:col10/1732743400077/Put/seqid=0 2024-11-27T21:36:40,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:40,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743460110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:40,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:40,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:40,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743460121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:40,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:40,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742363_1539 (size=12301) 2024-11-27T21:36:40,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743460124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:40,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1732743460125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:40,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743460127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:40,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:40,144 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,144 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,144 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:40,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:40,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:40,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743460233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:40,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:40,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743460233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:40,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:40,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:40,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-27T21:36:40,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:40,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:40,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:40,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:40,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:40,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:40,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:40,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1732743460253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:40,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:40,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743460253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:40,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:40,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:40,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743460254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:40,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:40,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:40,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-27T21:36:40,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-27T21:36:40,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:40,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:40,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:40,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:40,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:40,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:40,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:40,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743460456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:40,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:40,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743460456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:40,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:40,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743460464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:40,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:40,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:40,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:40,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:40,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1732743460467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:40,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:40,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743460467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:40,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:40,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:40,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/cf0f53912c5f487db482ae33bb30da53 2024-11-27T21:36:40,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/b9ce04f9e1f14334b3eb95f0a29fc800 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/b9ce04f9e1f14334b3eb95f0a29fc800 2024-11-27T21:36:40,539 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/b9ce04f9e1f14334b3eb95f0a29fc800, entries=200, sequenceid=390, filesize=14.4 K 2024-11-27T21:36:40,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/63342e8be8114ad2baa453ad7f14d7ed as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/63342e8be8114ad2baa453ad7f14d7ed 2024-11-27T21:36:40,550 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/63342e8be8114ad2baa453ad7f14d7ed, entries=150, sequenceid=390, filesize=12.0 K 2024-11-27T21:36:40,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/cf0f53912c5f487db482ae33bb30da53 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/cf0f53912c5f487db482ae33bb30da53 2024-11-27T21:36:40,557 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/cf0f53912c5f487db482ae33bb30da53, entries=150, sequenceid=390, filesize=12.0 K 2024-11-27T21:36:40,558 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for b39500f7a69456fd382cec5091ca715b in 480ms, sequenceid=390, compaction requested=false 2024-11-27T21:36:40,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:40,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-27T21:36:40,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:40,560 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2902): Flushing b39500f7a69456fd382cec5091ca715b 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-27T21:36:40,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/163968599f1345c388d5505ff0403d07 is 50, key is test_row_0/A:col10/1732743400125/Put/seqid=0 2024-11-27T21:36:40,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-27T21:36:40,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742364_1540 (size=12301) 2024-11-27T21:36:40,589 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/163968599f1345c388d5505ff0403d07 2024-11-27T21:36:40,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/47534800be7b4a0f88dd9a7f6e937cb0 is 50, key is test_row_0/B:col10/1732743400125/Put/seqid=0 2024-11-27T21:36:40,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742365_1541 (size=12301) 2024-11-27T21:36:40,626 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/47534800be7b4a0f88dd9a7f6e937cb0 2024-11-27T21:36:40,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/51105b2d6d4e45de9523d424aa4b1470 is 50, key is test_row_0/C:col10/1732743400125/Put/seqid=0 2024-11-27T21:36:40,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742366_1542 (size=12301) 2024-11-27T21:36:40,681 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/51105b2d6d4e45de9523d424aa4b1470 2024-11-27T21:36:40,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/163968599f1345c388d5505ff0403d07 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/163968599f1345c388d5505ff0403d07 2024-11-27T21:36:40,692 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/163968599f1345c388d5505ff0403d07, entries=150, sequenceid=409, filesize=12.0 K 2024-11-27T21:36:40,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/47534800be7b4a0f88dd9a7f6e937cb0 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/47534800be7b4a0f88dd9a7f6e937cb0 2024-11-27T21:36:40,697 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/47534800be7b4a0f88dd9a7f6e937cb0, entries=150, sequenceid=409, filesize=12.0 K 2024-11-27T21:36:40,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/51105b2d6d4e45de9523d424aa4b1470 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/51105b2d6d4e45de9523d424aa4b1470 2024-11-27T21:36:40,702 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/51105b2d6d4e45de9523d424aa4b1470, entries=150, sequenceid=409, filesize=12.0 K 2024-11-27T21:36:40,705 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=0 B/0 for b39500f7a69456fd382cec5091ca715b in 144ms, sequenceid=409, compaction requested=true 2024-11-27T21:36:40,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2603): Flush status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:40,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:40,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142 2024-11-27T21:36:40,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=142 2024-11-27T21:36:40,708 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-27T21:36:40,708 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 759 msec 2024-11-27T21:36:40,711 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees in 765 msec 2024-11-27T21:36:40,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on b39500f7a69456fd382cec5091ca715b 2024-11-27T21:36:40,791 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:36:40,791 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b39500f7a69456fd382cec5091ca715b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T21:36:40,798 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/af5b43ddd8534127b64e11b1a592394e is 50, key is test_row_0/A:col10/1732743400787/Put/seqid=0 2024-11-27T21:36:40,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742367_1543 (size=17181) 2024-11-27T21:36:40,808 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/af5b43ddd8534127b64e11b1a592394e 2024-11-27T21:36:40,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/cf8f635e61da443991051f9131e4af3c is 50, key is test_row_0/B:col10/1732743400787/Put/seqid=0 2024-11-27T21:36:40,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742368_1544 (size=12301) 2024-11-27T21:36:40,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:40,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743460855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:40,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:40,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:40,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743460856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:40,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743460856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:40,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:40,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:40,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:40,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1732743460857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:40,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:40,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743460872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:40,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:40,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:40,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:40,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:40,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743460973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:40,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:40,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743460973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:40,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:40,979 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,979 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,979 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:40,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:40,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743460983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:40,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:40,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:40,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743460983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:40,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:40,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1732743460983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:40,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:40,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:40,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:41,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-27T21:36:41,084 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:36:41,084 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:36:41,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees 2024-11-27T21:36:41,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-27T21:36:41,086 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:36:41,087 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:36:41,087 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:36:41,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:41,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743461184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:41,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:41,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743461184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:41,193 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:41,193 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:41,193 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:41,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:41,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:41,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:41,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-27T21:36:41,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:41,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743461194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:41,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:41,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:41,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:41,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:41,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743461203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:41,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:41,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1732743461203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:41,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:41,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:41,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:41,216 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:41,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:41,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:41,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-27T21:36:41,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:41,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:41,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:41,240 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:41,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:41,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:41,255 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/cf8f635e61da443991051f9131e4af3c 2024-11-27T21:36:41,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/799aeda6320a40f0824166a296257c71 is 50, key is test_row_0/C:col10/1732743400787/Put/seqid=0 2024-11-27T21:36:41,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742369_1545 (size=12301) 2024-11-27T21:36:41,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-27T21:36:41,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:41,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:41,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:41,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:41,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:41,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:41,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-27T21:36:41,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:41,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743461503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:41,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:41,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743461503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:41,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:41,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:41,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:41,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:41,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:41,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:41,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:41,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743461514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:41,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:41,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:41,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:41,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:41,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1732743461523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:41,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:41,531 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:41,531 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:41,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743461524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:41,531 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:41,532 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:41,532 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:41,532 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:41,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-27T21:36:41,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:41,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:41,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:41,562 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:41,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:41,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:41,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/799aeda6320a40f0824166a296257c71 2024-11-27T21:36:41,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/af5b43ddd8534127b64e11b1a592394e as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/af5b43ddd8534127b64e11b1a592394e 2024-11-27T21:36:41,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/af5b43ddd8534127b64e11b1a592394e, entries=250, sequenceid=420, filesize=16.8 K 2024-11-27T21:36:41,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/cf8f635e61da443991051f9131e4af3c as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/cf8f635e61da443991051f9131e4af3c 2024-11-27T21:36:41,681 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/cf8f635e61da443991051f9131e4af3c, entries=150, sequenceid=420, filesize=12.0 K 2024-11-27T21:36:41,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/799aeda6320a40f0824166a296257c71 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/799aeda6320a40f0824166a296257c71 2024-11-27T21:36:41,685 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/799aeda6320a40f0824166a296257c71, entries=150, sequenceid=420, filesize=12.0 K 2024-11-27T21:36:41,686 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for b39500f7a69456fd382cec5091ca715b in 895ms, sequenceid=420, compaction requested=true 2024-11-27T21:36:41,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:41,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b39500f7a69456fd382cec5091ca715b:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:36:41,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:41,686 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:36:41,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b39500f7a69456fd382cec5091ca715b:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:36:41,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:41,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b39500f7a69456fd382cec5091ca715b:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:36:41,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:36:41,686 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:36:41,687 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57240 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:36:41,687 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): b39500f7a69456fd382cec5091ca715b/A is initiating minor compaction (all files) 2024-11-27T21:36:41,687 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:36:41,687 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b39500f7a69456fd382cec5091ca715b/A in TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:41,687 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): b39500f7a69456fd382cec5091ca715b/B is initiating minor compaction (all files) 2024-11-27T21:36:41,687 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/75a6c69049b1418fb9598d761bdee04d, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/b9ce04f9e1f14334b3eb95f0a29fc800, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/163968599f1345c388d5505ff0403d07, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/af5b43ddd8534127b64e11b1a592394e] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp, totalSize=55.9 K 2024-11-27T21:36:41,687 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b39500f7a69456fd382cec5091ca715b/B in TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:41,687 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/f773ba42607c4dbba9315848bbf1a8f6, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/63342e8be8114ad2baa453ad7f14d7ed, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/47534800be7b4a0f88dd9a7f6e937cb0, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/cf8f635e61da443991051f9131e4af3c] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp, totalSize=48.8 K 2024-11-27T21:36:41,688 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75a6c69049b1418fb9598d761bdee04d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732743399059 2024-11-27T21:36:41,688 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting f773ba42607c4dbba9315848bbf1a8f6, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732743399059 2024-11-27T21:36:41,689 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9ce04f9e1f14334b3eb95f0a29fc800, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1732743399414 2024-11-27T21:36:41,689 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 63342e8be8114ad2baa453ad7f14d7ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1732743399414 2024-11-27T21:36:41,689 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 47534800be7b4a0f88dd9a7f6e937cb0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1732743400119 2024-11-27T21:36:41,689 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 163968599f1345c388d5505ff0403d07, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1732743400119 2024-11-27T21:36:41,689 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting af5b43ddd8534127b64e11b1a592394e, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1732743400780 2024-11-27T21:36:41,689 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting cf8f635e61da443991051f9131e4af3c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1732743400787 2024-11-27T21:36:41,697 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b39500f7a69456fd382cec5091ca715b#A#compaction#471 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:41,697 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/6ba86fbaecfc4491871111ba734fef3e is 50, key is test_row_0/A:col10/1732743400787/Put/seqid=0 2024-11-27T21:36:41,698 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b39500f7a69456fd382cec5091ca715b#B#compaction#472 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:41,698 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/c6d4a784afe946b39874a3df3ca07d90 is 50, key is test_row_0/B:col10/1732743400787/Put/seqid=0 2024-11-27T21:36:41,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742370_1546 (size=13153) 2024-11-27T21:36:41,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742371_1547 (size=13153) 2024-11-27T21:36:41,706 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/6ba86fbaecfc4491871111ba734fef3e as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/6ba86fbaecfc4491871111ba734fef3e 2024-11-27T21:36:41,710 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b39500f7a69456fd382cec5091ca715b/A of b39500f7a69456fd382cec5091ca715b into 6ba86fbaecfc4491871111ba734fef3e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:41,710 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:41,710 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., storeName=b39500f7a69456fd382cec5091ca715b/A, priority=12, startTime=1732743401686; duration=0sec 2024-11-27T21:36:41,710 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:36:41,710 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b39500f7a69456fd382cec5091ca715b:A 2024-11-27T21:36:41,710 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:36:41,710 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:36:41,710 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): b39500f7a69456fd382cec5091ca715b/C is initiating minor compaction (all files) 2024-11-27T21:36:41,711 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b39500f7a69456fd382cec5091ca715b/C in TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:41,711 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/0b9202d8eeb54541a6d275ef823bf366, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/cf0f53912c5f487db482ae33bb30da53, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/51105b2d6d4e45de9523d424aa4b1470, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/799aeda6320a40f0824166a296257c71] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp, totalSize=48.7 K 2024-11-27T21:36:41,711 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b9202d8eeb54541a6d275ef823bf366, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732743399059 2024-11-27T21:36:41,711 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf0f53912c5f487db482ae33bb30da53, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1732743399414 2024-11-27T21:36:41,711 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51105b2d6d4e45de9523d424aa4b1470, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1732743400119 2024-11-27T21:36:41,711 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 799aeda6320a40f0824166a296257c71, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1732743400787 2024-11-27T21:36:41,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-27T21:36:41,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-27T21:36:41,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:41,715 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2902): Flushing b39500f7a69456fd382cec5091ca715b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T21:36:41,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/a618be690812465797ec85188ea1c50e is 50, key is test_row_0/A:col10/1732743400852/Put/seqid=0 2024-11-27T21:36:41,720 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b39500f7a69456fd382cec5091ca715b#C#compaction#474 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:41,720 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/26bdaa24908244a49c1c78d926c136fd is 50, key is test_row_0/C:col10/1732743400787/Put/seqid=0 2024-11-27T21:36:41,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742373_1549 (size=13119) 2024-11-27T21:36:41,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742372_1548 (size=12301) 2024-11-27T21:36:42,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:42,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on b39500f7a69456fd382cec5091ca715b 2024-11-27T21:36:42,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:42,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1732743462043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:42,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:42,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743462047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:42,057 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:42,058 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:42,058 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:42,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:42,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:42,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:42,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:42,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743462049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:42,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:42,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:42,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:42,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:42,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743462057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:42,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:42,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:42,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:42,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:42,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743462059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:42,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:42,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:42,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:42,106 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/c6d4a784afe946b39874a3df3ca07d90 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/c6d4a784afe946b39874a3df3ca07d90 2024-11-27T21:36:42,111 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b39500f7a69456fd382cec5091ca715b/B of b39500f7a69456fd382cec5091ca715b into c6d4a784afe946b39874a3df3ca07d90(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:42,111 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:42,111 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., storeName=b39500f7a69456fd382cec5091ca715b/B, priority=12, startTime=1732743401686; duration=0sec 2024-11-27T21:36:42,111 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:42,111 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b39500f7a69456fd382cec5091ca715b:B 2024-11-27T21:36:42,129 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/26bdaa24908244a49c1c78d926c136fd as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/26bdaa24908244a49c1c78d926c136fd 2024-11-27T21:36:42,131 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/a618be690812465797ec85188ea1c50e 2024-11-27T21:36:42,136 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b39500f7a69456fd382cec5091ca715b/C of b39500f7a69456fd382cec5091ca715b into 26bdaa24908244a49c1c78d926c136fd(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:42,137 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:42,137 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., storeName=b39500f7a69456fd382cec5091ca715b/C, priority=12, startTime=1732743401686; duration=0sec 2024-11-27T21:36:42,137 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:42,137 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b39500f7a69456fd382cec5091ca715b:C 2024-11-27T21:36:42,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/8a4d433619c941afaa76d256f4773c8d is 50, key is test_row_0/B:col10/1732743400852/Put/seqid=0 2024-11-27T21:36:42,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742374_1550 (size=12301) 2024-11-27T21:36:42,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:42,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743462163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:42,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:42,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:42,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:42,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:42,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743462173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:42,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:42,182 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:42,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743462173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:42,182 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:42,182 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:42,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:42,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:42,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:42,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:42,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743462184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:42,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:42,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:42,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:42,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-27T21:36:42,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:42,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743462373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:42,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:42,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:42,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:42,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:42,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743462394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:42,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:42,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:42,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743462394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:42,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:42,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:42,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:42,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:42,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:42,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:42,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743462405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:42,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:42,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:42,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:42,544 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/8a4d433619c941afaa76d256f4773c8d 2024-11-27T21:36:42,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/ad40ee16298443a18401ef13c8951405 is 50, key is test_row_0/C:col10/1732743400852/Put/seqid=0 2024-11-27T21:36:42,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742375_1551 (size=12301) 2024-11-27T21:36:42,588 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/ad40ee16298443a18401ef13c8951405 2024-11-27T21:36:42,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/a618be690812465797ec85188ea1c50e as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/a618be690812465797ec85188ea1c50e 2024-11-27T21:36:42,620 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/a618be690812465797ec85188ea1c50e, entries=150, sequenceid=446, filesize=12.0 K 2024-11-27T21:36:42,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/8a4d433619c941afaa76d256f4773c8d as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/8a4d433619c941afaa76d256f4773c8d 2024-11-27T21:36:42,631 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/8a4d433619c941afaa76d256f4773c8d, entries=150, sequenceid=446, filesize=12.0 K 2024-11-27T21:36:42,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/ad40ee16298443a18401ef13c8951405 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/ad40ee16298443a18401ef13c8951405 2024-11-27T21:36:42,641 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/ad40ee16298443a18401ef13c8951405, entries=150, sequenceid=446, filesize=12.0 K 2024-11-27T21:36:42,645 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for b39500f7a69456fd382cec5091ca715b in 930ms, sequenceid=446, compaction requested=false 2024-11-27T21:36:42,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2603): Flush status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:42,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:42,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=144 2024-11-27T21:36:42,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=144 2024-11-27T21:36:42,649 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-27T21:36:42,649 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5600 sec 2024-11-27T21:36:42,654 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees in 1.5660 sec 2024-11-27T21:36:42,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on b39500f7a69456fd382cec5091ca715b 2024-11-27T21:36:42,692 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:36:42,692 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b39500f7a69456fd382cec5091ca715b 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-27T21:36:42,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/89ead217130c43f6b34ff17e4ece0ff8 is 50, key is test_row_0/A:col10/1732743402056/Put/seqid=0 2024-11-27T21:36:42,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742376_1552 (size=14741) 2024-11-27T21:36:42,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:42,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743462780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:42,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:42,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743462780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:42,794 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:42,794 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:42,794 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:42,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:42,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:42,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:42,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:42,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743462782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:42,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:42,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:42,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:42,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:42,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743462783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:42,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:42,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:42,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:42,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:42,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743462904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:42,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:42,913 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:42,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743462905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:42,913 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:42,913 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:42,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:42,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:42,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:42,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743462905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:42,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:42,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:42,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:42,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743462905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:42,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:42,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:42,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:42,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:42,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:43,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:43,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1732743463073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:43,084 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,39097,1732743267048 for 'test_row_1' in b39500f7a69456fd382cec5091ca715b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:36:43,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:43,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:43,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:43,125 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/89ead217130c43f6b34ff17e4ece0ff8 2024-11-27T21:36:43,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:43,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743463124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:43,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:43,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743463124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:43,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:43,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:43,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:43,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:43,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743463124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:43,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:43,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:43,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:43,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:43,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743463124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:43,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:43,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:43,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:43,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:43,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:43,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:43,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/b166e1e5ca584dca8e6e64a0b8a84ee3 is 50, key is test_row_0/B:col10/1732743402056/Put/seqid=0 2024-11-27T21:36:43,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742377_1553 (size=12301) 2024-11-27T21:36:43,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-27T21:36:43,244 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:36:43,245 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:36:43,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees 2024-11-27T21:36:43,249 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=145, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:36:43,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-27T21:36:43,251 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=145, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:36:43,251 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:36:43,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-27T21:36:43,387 INFO [Thread-2037 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2037. 2024-11-27T21:36:43,388 DEBUG [Thread-2037 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:36:43,388 DEBUG [Thread-2037 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:43,388 DEBUG [Thread-2037 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:43,388 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:36:43,391 INFO [Thread-2035 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2035. 2024-11-27T21:36:43,392 DEBUG [Thread-2035 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:36:43,392 DEBUG [Thread-2035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:43,392 DEBUG [Thread-2035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:43,392 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:36:43,396 INFO [Thread-2041 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2041. 2024-11-27T21:36:43,396 DEBUG [Thread-2041 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:36:43,396 DEBUG [Thread-2041 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:43,396 DEBUG [Thread-2041 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:43,396 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:36:43,400 INFO [Thread-2039 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2039. 2024-11-27T21:36:43,400 DEBUG [Thread-2039 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:36:43,400 DEBUG [Thread-2039 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:43,400 DEBUG [Thread-2039 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:43,400 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:36:43,402 INFO [Thread-2043 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2043. 2024-11-27T21:36:43,402 DEBUG [Thread-2043 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:36:43,402 DEBUG [Thread-2043 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:43,402 DEBUG [Thread-2043 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:43,402 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:36:43,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-27T21:36:43,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:43,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:43,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:43,405 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:43,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:43,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:43,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:43,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:43,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:43,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743463443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:43,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743463443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:43,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743463443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:43,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:43,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743463443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:43,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:43,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:43,444 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:43,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:43,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:43,444 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:43,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:43,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:43,444 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:43,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:43,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:43,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:43,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-27T21:36:43,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:43,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:43,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:43,558 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:43,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:43,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:43,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-27T21:36:43,594 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/b166e1e5ca584dca8e6e64a0b8a84ee3 2024-11-27T21:36:43,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/efd4a19b6398481fb8550c9d59428314 is 50, key is test_row_0/C:col10/1732743402056/Put/seqid=0 2024-11-27T21:36:43,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742378_1554 (size=12301) 2024-11-27T21:36:43,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-27T21:36:43,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:43,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:43,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:43,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:43,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:43,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:43,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-27T21:36:43,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:43,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:43,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:43,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:43,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:43,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:43,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-27T21:36:43,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:43,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:43,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:43,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39168 deadline: 1732743463953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:43,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1732743463953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:43,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39156 deadline: 1732743463953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:43,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:43,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:43,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39208 deadline: 1732743463953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:43,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:43,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:43,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:43,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:43,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:43,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:43,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:43,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:43,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 , the old value is region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:43,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:43,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., hostname=4585c39bed03,39097,1732743267048, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:36:44,003 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/efd4a19b6398481fb8550c9d59428314 2024-11-27T21:36:44,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/89ead217130c43f6b34ff17e4ece0ff8 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/89ead217130c43f6b34ff17e4ece0ff8 2024-11-27T21:36:44,009 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/89ead217130c43f6b34ff17e4ece0ff8, entries=200, sequenceid=461, filesize=14.4 K 2024-11-27T21:36:44,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/b166e1e5ca584dca8e6e64a0b8a84ee3 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/b166e1e5ca584dca8e6e64a0b8a84ee3 2024-11-27T21:36:44,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/b166e1e5ca584dca8e6e64a0b8a84ee3, entries=150, sequenceid=461, filesize=12.0 K 2024-11-27T21:36:44,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/efd4a19b6398481fb8550c9d59428314 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/efd4a19b6398481fb8550c9d59428314 2024-11-27T21:36:44,015 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/efd4a19b6398481fb8550c9d59428314, entries=150, sequenceid=461, filesize=12.0 K 2024-11-27T21:36:44,015 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for b39500f7a69456fd382cec5091ca715b in 1323ms, sequenceid=461, compaction requested=true 2024-11-27T21:36:44,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:44,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b39500f7a69456fd382cec5091ca715b:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:36:44,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:44,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b39500f7a69456fd382cec5091ca715b:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:36:44,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:44,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b39500f7a69456fd382cec5091ca715b:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:36:44,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:36:44,016 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:44,016 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:44,016 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:44,017 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): b39500f7a69456fd382cec5091ca715b/A is initiating minor compaction (all files) 2024-11-27T21:36:44,017 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b39500f7a69456fd382cec5091ca715b/A in TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:44,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-27T21:36:44,017 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/6ba86fbaecfc4491871111ba734fef3e, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/a618be690812465797ec85188ea1c50e, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/89ead217130c43f6b34ff17e4ece0ff8] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp, totalSize=39.3 K 2024-11-27T21:36:44,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:44,017 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2902): Flushing b39500f7a69456fd382cec5091ca715b 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-27T21:36:44,017 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:44,017 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ba86fbaecfc4491871111ba734fef3e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1732743400787 2024-11-27T21:36:44,017 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): b39500f7a69456fd382cec5091ca715b/B is initiating minor compaction (all files) 2024-11-27T21:36:44,017 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b39500f7a69456fd382cec5091ca715b/B in TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:44,017 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/c6d4a784afe946b39874a3df3ca07d90, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/8a4d433619c941afaa76d256f4773c8d, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/b166e1e5ca584dca8e6e64a0b8a84ee3] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp, totalSize=36.9 K 2024-11-27T21:36:44,017 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting a618be690812465797ec85188ea1c50e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1732743400837 2024-11-27T21:36:44,017 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting c6d4a784afe946b39874a3df3ca07d90, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1732743400787 2024-11-27T21:36:44,018 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89ead217130c43f6b34ff17e4ece0ff8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1732743402041 2024-11-27T21:36:44,018 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a4d433619c941afaa76d256f4773c8d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1732743400837 2024-11-27T21:36:44,018 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting b166e1e5ca584dca8e6e64a0b8a84ee3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1732743402056 2024-11-27T21:36:44,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/8fb55006bdf94333a08ddc6fdc786542 is 50, key is test_row_0/A:col10/1732743402781/Put/seqid=0 2024-11-27T21:36:44,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742379_1555 (size=12301) 2024-11-27T21:36:44,023 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b39500f7a69456fd382cec5091ca715b#A#compaction#481 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:44,023 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b39500f7a69456fd382cec5091ca715b#B#compaction#482 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:44,024 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/38aa8bc1b4db489caad522b7a89dce43 is 50, key is test_row_0/A:col10/1732743402056/Put/seqid=0 2024-11-27T21:36:44,024 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/3fddb284f2a8482f8ba3e97d4d9c9a1d is 50, key is test_row_0/B:col10/1732743402056/Put/seqid=0 2024-11-27T21:36:44,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742380_1556 (size=13255) 2024-11-27T21:36:44,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742381_1557 (size=13255) 2024-11-27T21:36:44,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-27T21:36:44,424 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/8fb55006bdf94333a08ddc6fdc786542 2024-11-27T21:36:44,430 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/38aa8bc1b4db489caad522b7a89dce43 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/38aa8bc1b4db489caad522b7a89dce43 2024-11-27T21:36:44,430 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/3fddb284f2a8482f8ba3e97d4d9c9a1d as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/3fddb284f2a8482f8ba3e97d4d9c9a1d 2024-11-27T21:36:44,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/d9563bf2fc51492c879d9ecc066fb725 is 50, key is test_row_0/B:col10/1732743402781/Put/seqid=0 2024-11-27T21:36:44,434 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b39500f7a69456fd382cec5091ca715b/A of b39500f7a69456fd382cec5091ca715b into 38aa8bc1b4db489caad522b7a89dce43(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:44,434 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b39500f7a69456fd382cec5091ca715b/B of b39500f7a69456fd382cec5091ca715b into 3fddb284f2a8482f8ba3e97d4d9c9a1d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:44,434 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:44,434 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:44,434 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., storeName=b39500f7a69456fd382cec5091ca715b/A, priority=13, startTime=1732743404016; duration=0sec 2024-11-27T21:36:44,434 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., storeName=b39500f7a69456fd382cec5091ca715b/B, priority=13, startTime=1732743404016; duration=0sec 2024-11-27T21:36:44,434 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:36:44,434 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b39500f7a69456fd382cec5091ca715b:A 2024-11-27T21:36:44,434 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:36:44,434 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b39500f7a69456fd382cec5091ca715b:B 2024-11-27T21:36:44,434 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:44,435 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:44,435 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): b39500f7a69456fd382cec5091ca715b/C is initiating minor compaction (all files) 2024-11-27T21:36:44,435 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b39500f7a69456fd382cec5091ca715b/C in TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:44,435 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/26bdaa24908244a49c1c78d926c136fd, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/ad40ee16298443a18401ef13c8951405, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/efd4a19b6398481fb8550c9d59428314] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp, totalSize=36.8 K 2024-11-27T21:36:44,435 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26bdaa24908244a49c1c78d926c136fd, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1732743400787 2024-11-27T21:36:44,436 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad40ee16298443a18401ef13c8951405, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1732743400837 2024-11-27T21:36:44,436 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting efd4a19b6398481fb8550c9d59428314, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1732743402056 2024-11-27T21:36:44,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742382_1558 (size=12301) 2024-11-27T21:36:44,442 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b39500f7a69456fd382cec5091ca715b#C#compaction#484 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:44,442 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/6c3f60beba904b80b6f15e4a4fe9720b is 50, key is test_row_0/C:col10/1732743402056/Put/seqid=0 2024-11-27T21:36:44,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742383_1559 (size=13221) 2024-11-27T21:36:44,453 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/6c3f60beba904b80b6f15e4a4fe9720b as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/6c3f60beba904b80b6f15e4a4fe9720b 2024-11-27T21:36:44,458 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b39500f7a69456fd382cec5091ca715b/C of b39500f7a69456fd382cec5091ca715b into 6c3f60beba904b80b6f15e4a4fe9720b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:44,458 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:44,458 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., storeName=b39500f7a69456fd382cec5091ca715b/C, priority=13, startTime=1732743404016; duration=0sec 2024-11-27T21:36:44,458 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:44,458 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b39500f7a69456fd382cec5091ca715b:C 2024-11-27T21:36:44,838 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/d9563bf2fc51492c879d9ecc066fb725 2024-11-27T21:36:44,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/bd2d76eba4bb4ab9b6118dc10cd958cf is 50, key is test_row_0/C:col10/1732743402781/Put/seqid=0 2024-11-27T21:36:44,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742384_1560 (size=12301) 2024-11-27T21:36:44,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on b39500f7a69456fd382cec5091ca715b 2024-11-27T21:36:44,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. as already flushing 2024-11-27T21:36:44,966 INFO [Thread-2026 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2026. 2024-11-27T21:36:44,966 INFO [Thread-2028 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2028. 2024-11-27T21:36:44,966 DEBUG [Thread-2026 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:36:44,966 DEBUG [Thread-2026 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:44,966 DEBUG [Thread-2028 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:36:44,966 DEBUG [Thread-2028 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:44,966 DEBUG [Thread-2026 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:44,966 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:36:44,966 DEBUG [Thread-2028 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:44,966 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:36:44,975 INFO [Thread-2030 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2030. 2024-11-27T21:36:44,975 INFO [Thread-2024 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2024. 2024-11-27T21:36:44,975 DEBUG [Thread-2024 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:36:44,975 DEBUG [Thread-2030 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:36:44,975 DEBUG [Thread-2024 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:44,975 DEBUG [Thread-2030 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:44,975 DEBUG [Thread-2030 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:44,975 DEBUG [Thread-2024 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:44,975 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:36:44,975 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:36:45,105 INFO [Thread-2032 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2032. 2024-11-27T21:36:45,105 DEBUG [Thread-2032 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:36:45,105 DEBUG [Thread-2032 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:45,105 DEBUG [Thread-2032 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:45,105 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:36:45,249 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/bd2d76eba4bb4ab9b6118dc10cd958cf 2024-11-27T21:36:45,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/8fb55006bdf94333a08ddc6fdc786542 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/8fb55006bdf94333a08ddc6fdc786542 2024-11-27T21:36:45,257 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/8fb55006bdf94333a08ddc6fdc786542, entries=150, sequenceid=484, filesize=12.0 K 2024-11-27T21:36:45,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/d9563bf2fc51492c879d9ecc066fb725 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/d9563bf2fc51492c879d9ecc066fb725 2024-11-27T21:36:45,260 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/d9563bf2fc51492c879d9ecc066fb725, entries=150, sequenceid=484, filesize=12.0 K 2024-11-27T21:36:45,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/bd2d76eba4bb4ab9b6118dc10cd958cf as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/bd2d76eba4bb4ab9b6118dc10cd958cf 2024-11-27T21:36:45,263 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/bd2d76eba4bb4ab9b6118dc10cd958cf, entries=150, sequenceid=484, filesize=12.0 K 2024-11-27T21:36:45,264 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=33.54 KB/34350 for b39500f7a69456fd382cec5091ca715b in 1247ms, sequenceid=484, compaction requested=false 2024-11-27T21:36:45,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2603): Flush status journal for b39500f7a69456fd382cec5091ca715b: 2024-11-27T21:36:45,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:45,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=146 2024-11-27T21:36:45,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=146 2024-11-27T21:36:45,267 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-27T21:36:45,267 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0140 sec 2024-11-27T21:36:45,268 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees in 2.0220 sec 2024-11-27T21:36:45,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-27T21:36:45,394 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:36:45,394 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-27T21:36:45,394 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-11-27T21:36:45,394 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2024-11-27T21:36:45,394 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-11-27T21:36:45,394 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-27T21:36:45,394 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-27T21:36:45,394 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-27T21:36:45,394 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-27T21:36:45,394 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1209 2024-11-27T21:36:45,394 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3626 rows 2024-11-27T21:36:45,394 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1217 2024-11-27T21:36:45,394 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3638 rows 2024-11-27T21:36:45,394 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1215 2024-11-27T21:36:45,394 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3627 rows 2024-11-27T21:36:45,394 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1206 2024-11-27T21:36:45,394 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3614 rows 2024-11-27T21:36:45,394 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1215 2024-11-27T21:36:45,394 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3635 rows 2024-11-27T21:36:45,394 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T21:36:45,394 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T21:36:45,394 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:45,394 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:45,394 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:36:45,395 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-27T21:36:45,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-27T21:36:45,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-27T21:36:45,398 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732743405398"}]},"ts":"1732743405398"} 2024-11-27T21:36:45,399 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-27T21:36:45,399 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-27T21:36:45,399 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T21:36:45,401 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b39500f7a69456fd382cec5091ca715b, UNASSIGN}] 2024-11-27T21:36:45,401 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b39500f7a69456fd382cec5091ca715b, UNASSIGN 2024-11-27T21:36:45,402 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=b39500f7a69456fd382cec5091ca715b, regionState=CLOSING, regionLocation=4585c39bed03,39097,1732743267048 2024-11-27T21:36:45,403 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b39500f7a69456fd382cec5091ca715b, UNASSIGN because future has completed 2024-11-27T21:36:45,403 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T21:36:45,403 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048}] 2024-11-27T21:36:45,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-27T21:36:45,555 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(122): Close b39500f7a69456fd382cec5091ca715b 2024-11-27T21:36:45,555 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T21:36:45,555 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1722): Closing b39500f7a69456fd382cec5091ca715b, disabling compactions & flushes 2024-11-27T21:36:45,555 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:45,555 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:45,555 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. after waiting 0 ms 2024-11-27T21:36:45,555 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:45,555 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(2902): Flushing b39500f7a69456fd382cec5091ca715b 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-27T21:36:45,559 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/adfe7c52065948b78a081c52218b1c66 is 50, key is test_row_0/A:col10/1732743404974/Put/seqid=0 2024-11-27T21:36:45,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742385_1561 (size=9857) 2024-11-27T21:36:45,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-27T21:36:45,962 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=495 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/adfe7c52065948b78a081c52218b1c66 2024-11-27T21:36:45,968 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/3a5e8ef27fe14a2788a1715b3f21ada0 is 50, key is test_row_0/B:col10/1732743404974/Put/seqid=0 2024-11-27T21:36:45,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742386_1562 (size=9857) 2024-11-27T21:36:46,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-27T21:36:46,371 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=495 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/3a5e8ef27fe14a2788a1715b3f21ada0 2024-11-27T21:36:46,376 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/1e6c50981e394badaeff058f276d3edf is 50, key is test_row_0/C:col10/1732743404974/Put/seqid=0 2024-11-27T21:36:46,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742387_1563 (size=9857) 2024-11-27T21:36:46,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-27T21:36:46,780 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=495 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/1e6c50981e394badaeff058f276d3edf 2024-11-27T21:36:46,783 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/A/adfe7c52065948b78a081c52218b1c66 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/adfe7c52065948b78a081c52218b1c66 2024-11-27T21:36:46,786 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/adfe7c52065948b78a081c52218b1c66, entries=100, sequenceid=495, filesize=9.6 K 2024-11-27T21:36:46,786 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/B/3a5e8ef27fe14a2788a1715b3f21ada0 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/3a5e8ef27fe14a2788a1715b3f21ada0 2024-11-27T21:36:46,789 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/3a5e8ef27fe14a2788a1715b3f21ada0, entries=100, sequenceid=495, filesize=9.6 K 2024-11-27T21:36:46,790 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/.tmp/C/1e6c50981e394badaeff058f276d3edf as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/1e6c50981e394badaeff058f276d3edf 2024-11-27T21:36:46,792 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/1e6c50981e394badaeff058f276d3edf, entries=100, sequenceid=495, filesize=9.6 K 2024-11-27T21:36:46,793 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for b39500f7a69456fd382cec5091ca715b in 1238ms, sequenceid=495, compaction requested=true 2024-11-27T21:36:46,793 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/1300846a9e154202b504b26f03cf9c1d, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/bf4732f2e7e64ec0ba54b960976f0a4a, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/1f66fa5728d641b585ffa51789513f45, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/0978edafe3ae40959ae32c53c5fa6650, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/72febdd05ec8447b8c13f7cae7e43b83, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/136f17a68a984159a242b7ade09408cc, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/788e4d09b4b64f3bbe8c4156f0c06f45, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/6ab2c2d1767c485680b562167b7ea6f3, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/378c98f42aae468e9eee9e5fce05a089, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/b6eb3ab0845a4209930b74e1057a420c, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/ab52b5611c7a4d9a8fd9cdb4a2c529f1, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/6c341149875e47a6bbd139257b025c56, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/00bad1011e84413192a3cb697f434bf7, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/1bc5ef6311974b4c9201b900c0b6f1ee, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/405c58fdd007435082c0ebd9b48a766e, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/efad840aba294fd493300574b5b78f74, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/2b8618b677a74c98a10582413bcb7fd2, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/89898fd9930a4997bcadda89260ea3bd, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/280cdffa3760460c9121b1b67b7cd007, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/3c48bdc98c0542aa86533148520c196f, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/b913858e7f404dc3b0a632e552268e68, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/75a6c69049b1418fb9598d761bdee04d, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/b9ce04f9e1f14334b3eb95f0a29fc800, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/163968599f1345c388d5505ff0403d07, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/af5b43ddd8534127b64e11b1a592394e, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/6ba86fbaecfc4491871111ba734fef3e, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/a618be690812465797ec85188ea1c50e, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/89ead217130c43f6b34ff17e4ece0ff8] to archive 2024-11-27T21:36:46,794 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T21:36:46,795 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/1300846a9e154202b504b26f03cf9c1d to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/1300846a9e154202b504b26f03cf9c1d 2024-11-27T21:36:46,796 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/bf4732f2e7e64ec0ba54b960976f0a4a to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/bf4732f2e7e64ec0ba54b960976f0a4a 2024-11-27T21:36:46,796 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/1f66fa5728d641b585ffa51789513f45 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/1f66fa5728d641b585ffa51789513f45 2024-11-27T21:36:46,797 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/0978edafe3ae40959ae32c53c5fa6650 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/0978edafe3ae40959ae32c53c5fa6650 2024-11-27T21:36:46,798 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/72febdd05ec8447b8c13f7cae7e43b83 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/72febdd05ec8447b8c13f7cae7e43b83 2024-11-27T21:36:46,798 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/136f17a68a984159a242b7ade09408cc to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/136f17a68a984159a242b7ade09408cc 2024-11-27T21:36:46,799 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/788e4d09b4b64f3bbe8c4156f0c06f45 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/788e4d09b4b64f3bbe8c4156f0c06f45 2024-11-27T21:36:46,800 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/6ab2c2d1767c485680b562167b7ea6f3 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/6ab2c2d1767c485680b562167b7ea6f3 2024-11-27T21:36:46,801 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/378c98f42aae468e9eee9e5fce05a089 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/378c98f42aae468e9eee9e5fce05a089 2024-11-27T21:36:46,802 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/b6eb3ab0845a4209930b74e1057a420c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/b6eb3ab0845a4209930b74e1057a420c 2024-11-27T21:36:46,802 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/ab52b5611c7a4d9a8fd9cdb4a2c529f1 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/ab52b5611c7a4d9a8fd9cdb4a2c529f1 2024-11-27T21:36:46,803 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/6c341149875e47a6bbd139257b025c56 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/6c341149875e47a6bbd139257b025c56 2024-11-27T21:36:46,804 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/00bad1011e84413192a3cb697f434bf7 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/00bad1011e84413192a3cb697f434bf7 2024-11-27T21:36:46,804 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/1bc5ef6311974b4c9201b900c0b6f1ee to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/1bc5ef6311974b4c9201b900c0b6f1ee 2024-11-27T21:36:46,805 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/405c58fdd007435082c0ebd9b48a766e to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/405c58fdd007435082c0ebd9b48a766e 2024-11-27T21:36:46,806 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/efad840aba294fd493300574b5b78f74 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/efad840aba294fd493300574b5b78f74 2024-11-27T21:36:46,806 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/2b8618b677a74c98a10582413bcb7fd2 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/2b8618b677a74c98a10582413bcb7fd2 2024-11-27T21:36:46,807 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/89898fd9930a4997bcadda89260ea3bd to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/89898fd9930a4997bcadda89260ea3bd 2024-11-27T21:36:46,808 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/280cdffa3760460c9121b1b67b7cd007 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/280cdffa3760460c9121b1b67b7cd007 2024-11-27T21:36:46,809 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/3c48bdc98c0542aa86533148520c196f to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/3c48bdc98c0542aa86533148520c196f 2024-11-27T21:36:46,810 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/b913858e7f404dc3b0a632e552268e68 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/b913858e7f404dc3b0a632e552268e68 2024-11-27T21:36:46,810 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/75a6c69049b1418fb9598d761bdee04d to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/75a6c69049b1418fb9598d761bdee04d 2024-11-27T21:36:46,811 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/b9ce04f9e1f14334b3eb95f0a29fc800 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/b9ce04f9e1f14334b3eb95f0a29fc800 2024-11-27T21:36:46,812 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/163968599f1345c388d5505ff0403d07 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/163968599f1345c388d5505ff0403d07 2024-11-27T21:36:46,813 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/af5b43ddd8534127b64e11b1a592394e to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/af5b43ddd8534127b64e11b1a592394e 2024-11-27T21:36:46,814 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/6ba86fbaecfc4491871111ba734fef3e to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/6ba86fbaecfc4491871111ba734fef3e 2024-11-27T21:36:46,815 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/a618be690812465797ec85188ea1c50e to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/a618be690812465797ec85188ea1c50e 2024-11-27T21:36:46,816 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/89ead217130c43f6b34ff17e4ece0ff8 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/89ead217130c43f6b34ff17e4ece0ff8 2024-11-27T21:36:46,817 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/a90203520a384fada847094c52d14e7d, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/ae0687cc309846c4ad995dd3cc4f42e5, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/3d3c1c8850f2424cb72ef6012c9cded7, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/33d0a182aec2445c956242a7f988129f, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/78bfd9bf375c41d3a0ece5b78aab4337, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/eb6b298c0eb14de180aebae076a8a4d7, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/01046d8c4b0749e4b51e044f2d97013d, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/d5ab9bff5d354260bd69a5f7c15bd1a6, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/260d3499d54f424eabb572b7ca3a79dc, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/c575e90fea5346f4ad41769b459cb5b6, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/049403e8621147a0907b873b31c4ca05, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/fe75974c517d4aaaa4ea0d6b01ec240d, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/e19d73b7a51047e4bf6dca9674dcb859, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/402a13f446124c4daecdd2ad12999f0d, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/86318238ea354f5fab4ad744b914e8d2, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/f51a8690ae3e4033b699eb675f53f2ec, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/c108ec03c0c2479eaa74110d850e73bd, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/467b8e85727245c4ae7744a07d8fbeb8, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/081123bcbd4b41f8b783856faae553cb, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/9d33aad173344bed8dc394388f02cb00, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/f773ba42607c4dbba9315848bbf1a8f6, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/2fd5cfeeecfb41f4a400cf7eae01a466, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/63342e8be8114ad2baa453ad7f14d7ed, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/47534800be7b4a0f88dd9a7f6e937cb0, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/c6d4a784afe946b39874a3df3ca07d90, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/cf8f635e61da443991051f9131e4af3c, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/8a4d433619c941afaa76d256f4773c8d, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/b166e1e5ca584dca8e6e64a0b8a84ee3] to archive 2024-11-27T21:36:46,818 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T21:36:46,819 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/a90203520a384fada847094c52d14e7d to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/a90203520a384fada847094c52d14e7d 2024-11-27T21:36:46,820 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/ae0687cc309846c4ad995dd3cc4f42e5 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/ae0687cc309846c4ad995dd3cc4f42e5 2024-11-27T21:36:46,820 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/3d3c1c8850f2424cb72ef6012c9cded7 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/3d3c1c8850f2424cb72ef6012c9cded7 2024-11-27T21:36:46,821 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/33d0a182aec2445c956242a7f988129f to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/33d0a182aec2445c956242a7f988129f 2024-11-27T21:36:46,822 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/78bfd9bf375c41d3a0ece5b78aab4337 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/78bfd9bf375c41d3a0ece5b78aab4337 2024-11-27T21:36:46,823 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/eb6b298c0eb14de180aebae076a8a4d7 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/eb6b298c0eb14de180aebae076a8a4d7 2024-11-27T21:36:46,824 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/01046d8c4b0749e4b51e044f2d97013d to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/01046d8c4b0749e4b51e044f2d97013d 2024-11-27T21:36:46,824 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/d5ab9bff5d354260bd69a5f7c15bd1a6 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/d5ab9bff5d354260bd69a5f7c15bd1a6 2024-11-27T21:36:46,825 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/260d3499d54f424eabb572b7ca3a79dc to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/260d3499d54f424eabb572b7ca3a79dc 2024-11-27T21:36:46,826 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/c575e90fea5346f4ad41769b459cb5b6 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/c575e90fea5346f4ad41769b459cb5b6 2024-11-27T21:36:46,826 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/049403e8621147a0907b873b31c4ca05 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/049403e8621147a0907b873b31c4ca05 2024-11-27T21:36:46,827 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/fe75974c517d4aaaa4ea0d6b01ec240d to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/fe75974c517d4aaaa4ea0d6b01ec240d 2024-11-27T21:36:46,828 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/e19d73b7a51047e4bf6dca9674dcb859 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/e19d73b7a51047e4bf6dca9674dcb859 2024-11-27T21:36:46,829 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/402a13f446124c4daecdd2ad12999f0d to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/402a13f446124c4daecdd2ad12999f0d 2024-11-27T21:36:46,829 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/86318238ea354f5fab4ad744b914e8d2 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/86318238ea354f5fab4ad744b914e8d2 2024-11-27T21:36:46,830 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/f51a8690ae3e4033b699eb675f53f2ec to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/f51a8690ae3e4033b699eb675f53f2ec 2024-11-27T21:36:46,831 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/c108ec03c0c2479eaa74110d850e73bd to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/c108ec03c0c2479eaa74110d850e73bd 2024-11-27T21:36:46,831 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/467b8e85727245c4ae7744a07d8fbeb8 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/467b8e85727245c4ae7744a07d8fbeb8 2024-11-27T21:36:46,832 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/081123bcbd4b41f8b783856faae553cb to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/081123bcbd4b41f8b783856faae553cb 2024-11-27T21:36:46,833 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/9d33aad173344bed8dc394388f02cb00 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/9d33aad173344bed8dc394388f02cb00 2024-11-27T21:36:46,833 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/f773ba42607c4dbba9315848bbf1a8f6 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/f773ba42607c4dbba9315848bbf1a8f6 2024-11-27T21:36:46,834 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/2fd5cfeeecfb41f4a400cf7eae01a466 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/2fd5cfeeecfb41f4a400cf7eae01a466 2024-11-27T21:36:46,835 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/63342e8be8114ad2baa453ad7f14d7ed to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/63342e8be8114ad2baa453ad7f14d7ed 2024-11-27T21:36:46,835 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/47534800be7b4a0f88dd9a7f6e937cb0 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/47534800be7b4a0f88dd9a7f6e937cb0 2024-11-27T21:36:46,836 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/c6d4a784afe946b39874a3df3ca07d90 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/c6d4a784afe946b39874a3df3ca07d90 2024-11-27T21:36:46,837 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/cf8f635e61da443991051f9131e4af3c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/cf8f635e61da443991051f9131e4af3c 2024-11-27T21:36:46,838 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/8a4d433619c941afaa76d256f4773c8d to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/8a4d433619c941afaa76d256f4773c8d 2024-11-27T21:36:46,838 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/b166e1e5ca584dca8e6e64a0b8a84ee3 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/b166e1e5ca584dca8e6e64a0b8a84ee3 2024-11-27T21:36:46,840 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/82417af6b37d4df7ad217871d8deb194, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/389e4cfe0b444c1aa46ceb3ac89f841c, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/8f0018b80f2b476fae9a374515250390, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/225d5440cfe241d7987ce9f285d973fa, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/5c57060483ca429c92fba104086eedbd, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/45e60194b9074f328f6a5d2ebcb9057f, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/9b7fffbbe2f443ffaa10e7997c06d0ae, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/77981dbfb44f42a0b728938f5d5e2774, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/578235b776494336b63216e1efc114bd, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/2e663844ed264e3993c9ad55a44d35c3, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/bc5be490b9824866a5706a785773f936, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/a681f9365014485cba32b31f252b272c, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/f35e78358c364ce48a61ff061697457e, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/227b1b1f04ce45aa8d1431594781b7c3, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/5ff8b26f8c1e4c2a8b3224c2cd31abc9, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/d05aafdc1b694d738b9bcde5bb13f109, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/2afc13f6865c45649b61e6995219dfad, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/b79b9965e40240a586fe0017c4faa4be, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/027b6a664210432391bd31eac56c8657, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/0b9202d8eeb54541a6d275ef823bf366, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/7a02a2bbe66a43c5a7c25db1bcd68eab, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/cf0f53912c5f487db482ae33bb30da53, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/51105b2d6d4e45de9523d424aa4b1470, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/26bdaa24908244a49c1c78d926c136fd, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/799aeda6320a40f0824166a296257c71, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/ad40ee16298443a18401ef13c8951405, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/efd4a19b6398481fb8550c9d59428314] to archive 2024-11-27T21:36:46,840 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T21:36:46,842 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/82417af6b37d4df7ad217871d8deb194 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/82417af6b37d4df7ad217871d8deb194 2024-11-27T21:36:46,842 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/389e4cfe0b444c1aa46ceb3ac89f841c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/389e4cfe0b444c1aa46ceb3ac89f841c 2024-11-27T21:36:46,843 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/8f0018b80f2b476fae9a374515250390 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/8f0018b80f2b476fae9a374515250390 2024-11-27T21:36:46,844 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/225d5440cfe241d7987ce9f285d973fa to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/225d5440cfe241d7987ce9f285d973fa 2024-11-27T21:36:46,844 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/5c57060483ca429c92fba104086eedbd to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/5c57060483ca429c92fba104086eedbd 2024-11-27T21:36:46,845 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/45e60194b9074f328f6a5d2ebcb9057f to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/45e60194b9074f328f6a5d2ebcb9057f 2024-11-27T21:36:46,846 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/9b7fffbbe2f443ffaa10e7997c06d0ae to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/9b7fffbbe2f443ffaa10e7997c06d0ae 2024-11-27T21:36:46,847 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/77981dbfb44f42a0b728938f5d5e2774 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/77981dbfb44f42a0b728938f5d5e2774 2024-11-27T21:36:46,847 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/578235b776494336b63216e1efc114bd to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/578235b776494336b63216e1efc114bd 2024-11-27T21:36:46,848 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/2e663844ed264e3993c9ad55a44d35c3 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/2e663844ed264e3993c9ad55a44d35c3 2024-11-27T21:36:46,849 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/bc5be490b9824866a5706a785773f936 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/bc5be490b9824866a5706a785773f936 2024-11-27T21:36:46,849 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/a681f9365014485cba32b31f252b272c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/a681f9365014485cba32b31f252b272c 2024-11-27T21:36:46,850 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/f35e78358c364ce48a61ff061697457e to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/f35e78358c364ce48a61ff061697457e 2024-11-27T21:36:46,851 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/227b1b1f04ce45aa8d1431594781b7c3 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/227b1b1f04ce45aa8d1431594781b7c3 2024-11-27T21:36:46,852 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/5ff8b26f8c1e4c2a8b3224c2cd31abc9 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/5ff8b26f8c1e4c2a8b3224c2cd31abc9 2024-11-27T21:36:46,852 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/d05aafdc1b694d738b9bcde5bb13f109 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/d05aafdc1b694d738b9bcde5bb13f109 2024-11-27T21:36:46,853 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/2afc13f6865c45649b61e6995219dfad to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/2afc13f6865c45649b61e6995219dfad 2024-11-27T21:36:46,855 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/b79b9965e40240a586fe0017c4faa4be to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/b79b9965e40240a586fe0017c4faa4be 2024-11-27T21:36:46,856 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/027b6a664210432391bd31eac56c8657 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/027b6a664210432391bd31eac56c8657 2024-11-27T21:36:46,856 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/0b9202d8eeb54541a6d275ef823bf366 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/0b9202d8eeb54541a6d275ef823bf366 2024-11-27T21:36:46,857 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/7a02a2bbe66a43c5a7c25db1bcd68eab to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/7a02a2bbe66a43c5a7c25db1bcd68eab 2024-11-27T21:36:46,858 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/cf0f53912c5f487db482ae33bb30da53 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/cf0f53912c5f487db482ae33bb30da53 2024-11-27T21:36:46,858 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/51105b2d6d4e45de9523d424aa4b1470 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/51105b2d6d4e45de9523d424aa4b1470 2024-11-27T21:36:46,859 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/26bdaa24908244a49c1c78d926c136fd to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/26bdaa24908244a49c1c78d926c136fd 2024-11-27T21:36:46,860 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/799aeda6320a40f0824166a296257c71 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/799aeda6320a40f0824166a296257c71 2024-11-27T21:36:46,861 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/ad40ee16298443a18401ef13c8951405 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/ad40ee16298443a18401ef13c8951405 2024-11-27T21:36:46,861 DEBUG [StoreCloser-TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/efd4a19b6398481fb8550c9d59428314 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/efd4a19b6398481fb8550c9d59428314 2024-11-27T21:36:46,865 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/recovered.edits/498.seqid, newMaxSeqId=498, maxSeqId=1 2024-11-27T21:36:46,866 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. 2024-11-27T21:36:46,866 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1676): Region close journal for b39500f7a69456fd382cec5091ca715b: Waiting for close lock at 1732743405555Running coprocessor pre-close hooks at 1732743405555Disabling compacts and flushes for region at 1732743405555Disabling writes for close at 1732743405555Obtaining lock to block concurrent updates at 1732743405555Preparing flush snapshotting stores in b39500f7a69456fd382cec5091ca715b at 1732743405555Finished memstore snapshotting TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732743405556 (+1 ms)Flushing stores of TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b. at 1732743405556Flushing b39500f7a69456fd382cec5091ca715b/A: creating writer at 1732743405556Flushing b39500f7a69456fd382cec5091ca715b/A: appending metadata at 1732743405558 (+2 ms)Flushing b39500f7a69456fd382cec5091ca715b/A: closing flushed file at 1732743405559 (+1 ms)Flushing b39500f7a69456fd382cec5091ca715b/B: creating writer at 1732743405965 (+406 ms)Flushing b39500f7a69456fd382cec5091ca715b/B: appending metadata at 1732743405967 (+2 ms)Flushing b39500f7a69456fd382cec5091ca715b/B: closing flushed file at 1732743405967Flushing b39500f7a69456fd382cec5091ca715b/C: creating writer at 1732743406374 (+407 ms)Flushing b39500f7a69456fd382cec5091ca715b/C: appending metadata at 1732743406376 (+2 ms)Flushing b39500f7a69456fd382cec5091ca715b/C: closing flushed file at 1732743406376Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2974f269: reopening flushed file at 1732743406783 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@161567a4: reopening flushed file at 1732743406786 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@41d36e38: reopening flushed file at 1732743406789 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for b39500f7a69456fd382cec5091ca715b in 1238ms, sequenceid=495, compaction requested=true at 1732743406793 (+4 ms)Writing region close event to WAL at 1732743406863 (+70 ms)Running coprocessor post-close hooks at 1732743406866 (+3 ms)Closed at 1732743406866 2024-11-27T21:36:46,868 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(157): Closed b39500f7a69456fd382cec5091ca715b 2024-11-27T21:36:46,868 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=b39500f7a69456fd382cec5091ca715b, regionState=CLOSED 2024-11-27T21:36:46,870 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 because future has completed 2024-11-27T21:36:46,872 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-27T21:36:46,872 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; CloseRegionProcedure b39500f7a69456fd382cec5091ca715b, server=4585c39bed03,39097,1732743267048 in 1.4670 sec 2024-11-27T21:36:46,873 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-27T21:36:46,873 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b39500f7a69456fd382cec5091ca715b, UNASSIGN in 1.4720 sec 2024-11-27T21:36:46,875 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-27T21:36:46,875 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4740 sec 2024-11-27T21:36:46,876 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732743406875"}]},"ts":"1732743406875"} 2024-11-27T21:36:46,877 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-27T21:36:46,877 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-27T21:36:46,878 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4820 sec 2024-11-27T21:36:47,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-27T21:36:47,543 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:36:47,544 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-27T21:36:47,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=151, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:36:47,545 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=151, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:36:47,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-27T21:36:47,546 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=151, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:36:47,548 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b 2024-11-27T21:36:47,550 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A, FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B, FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C, FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/recovered.edits] 2024-11-27T21:36:47,552 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/38aa8bc1b4db489caad522b7a89dce43 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/38aa8bc1b4db489caad522b7a89dce43 2024-11-27T21:36:47,553 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/8fb55006bdf94333a08ddc6fdc786542 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/8fb55006bdf94333a08ddc6fdc786542 2024-11-27T21:36:47,554 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/adfe7c52065948b78a081c52218b1c66 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/A/adfe7c52065948b78a081c52218b1c66 2024-11-27T21:36:47,555 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/3a5e8ef27fe14a2788a1715b3f21ada0 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/3a5e8ef27fe14a2788a1715b3f21ada0 2024-11-27T21:36:47,556 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/3fddb284f2a8482f8ba3e97d4d9c9a1d to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/3fddb284f2a8482f8ba3e97d4d9c9a1d 2024-11-27T21:36:47,557 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/d9563bf2fc51492c879d9ecc066fb725 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/B/d9563bf2fc51492c879d9ecc066fb725 2024-11-27T21:36:47,558 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/1e6c50981e394badaeff058f276d3edf to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/1e6c50981e394badaeff058f276d3edf 2024-11-27T21:36:47,559 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/6c3f60beba904b80b6f15e4a4fe9720b to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/6c3f60beba904b80b6f15e4a4fe9720b 2024-11-27T21:36:47,560 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/bd2d76eba4bb4ab9b6118dc10cd958cf to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/C/bd2d76eba4bb4ab9b6118dc10cd958cf 2024-11-27T21:36:47,562 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/recovered.edits/498.seqid to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b/recovered.edits/498.seqid 2024-11-27T21:36:47,562 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/b39500f7a69456fd382cec5091ca715b 2024-11-27T21:36:47,562 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-27T21:36:47,564 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=151, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:36:47,566 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-27T21:36:47,567 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-27T21:36:47,568 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=151, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:36:47,568 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-27T21:36:47,568 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732743407568"}]},"ts":"9223372036854775807"} 2024-11-27T21:36:47,570 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-27T21:36:47,570 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => b39500f7a69456fd382cec5091ca715b, NAME => 'TestAcidGuarantees,,1732743381180.b39500f7a69456fd382cec5091ca715b.', STARTKEY => '', ENDKEY => ''}] 2024-11-27T21:36:47,570 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-27T21:36:47,570 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732743407570"}]},"ts":"9223372036854775807"} 2024-11-27T21:36:47,571 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-27T21:36:47,571 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=151, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:36:47,572 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 27 msec 2024-11-27T21:36:47,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-27T21:36:47,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-27T21:36:47,654 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:36:47,663 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=240 (was 240), OpenFileDescriptor=449 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=360 (was 308) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8257 (was 8248) - AvailableMemoryMB LEAK? - 2024-11-27T21:36:47,671 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=240, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=360, ProcessCount=11, AvailableMemoryMB=8256 2024-11-27T21:36:47,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T21:36:47,672 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T21:36:47,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-27T21:36:47,674 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T21:36:47,674 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:36:47,674 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 152 2024-11-27T21:36:47,675 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T21:36:47,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-27T21:36:47,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742388_1564 (size=959) 2024-11-27T21:36:47,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-27T21:36:47,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-27T21:36:48,081 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d 2024-11-27T21:36:48,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742389_1565 (size=53) 2024-11-27T21:36:48,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-27T21:36:48,486 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T21:36:48,486 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 3276e02ae096deef8165bc8c50c74666, disabling compactions & flushes 2024-11-27T21:36:48,486 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:48,486 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:48,486 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. after waiting 0 ms 2024-11-27T21:36:48,486 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:48,486 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:48,486 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3276e02ae096deef8165bc8c50c74666: Waiting for close lock at 1732743408486Disabling compacts and flushes for region at 1732743408486Disabling writes for close at 1732743408486Writing region close event to WAL at 1732743408486Closed at 1732743408486 2024-11-27T21:36:48,487 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T21:36:48,488 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732743408487"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732743408487"}]},"ts":"1732743408487"} 2024-11-27T21:36:48,489 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T21:36:48,490 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T21:36:48,490 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732743408490"}]},"ts":"1732743408490"} 2024-11-27T21:36:48,491 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-27T21:36:48,492 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3276e02ae096deef8165bc8c50c74666, ASSIGN}] 2024-11-27T21:36:48,493 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3276e02ae096deef8165bc8c50c74666, ASSIGN 2024-11-27T21:36:48,494 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3276e02ae096deef8165bc8c50c74666, ASSIGN; state=OFFLINE, location=4585c39bed03,39097,1732743267048; forceNewPlan=false, retain=false 2024-11-27T21:36:48,644 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=3276e02ae096deef8165bc8c50c74666, regionState=OPENING, regionLocation=4585c39bed03,39097,1732743267048 2024-11-27T21:36:48,646 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3276e02ae096deef8165bc8c50c74666, ASSIGN because future has completed 2024-11-27T21:36:48,646 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048}] 2024-11-27T21:36:48,800 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:48,800 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(7752): Opening region: {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} 2024-11-27T21:36:48,801 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3276e02ae096deef8165bc8c50c74666 2024-11-27T21:36:48,801 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T21:36:48,801 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(7794): checking encryption for 3276e02ae096deef8165bc8c50c74666 2024-11-27T21:36:48,801 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(7797): checking classloading for 3276e02ae096deef8165bc8c50c74666 2024-11-27T21:36:48,802 INFO [StoreOpener-3276e02ae096deef8165bc8c50c74666-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3276e02ae096deef8165bc8c50c74666 2024-11-27T21:36:48,803 INFO [StoreOpener-3276e02ae096deef8165bc8c50c74666-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3276e02ae096deef8165bc8c50c74666 columnFamilyName A 2024-11-27T21:36:48,803 DEBUG [StoreOpener-3276e02ae096deef8165bc8c50c74666-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:36:48,803 INFO [StoreOpener-3276e02ae096deef8165bc8c50c74666-1 {}] regionserver.HStore(327): Store=3276e02ae096deef8165bc8c50c74666/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:36:48,803 INFO [StoreOpener-3276e02ae096deef8165bc8c50c74666-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3276e02ae096deef8165bc8c50c74666 2024-11-27T21:36:48,804 INFO [StoreOpener-3276e02ae096deef8165bc8c50c74666-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3276e02ae096deef8165bc8c50c74666 columnFamilyName B 2024-11-27T21:36:48,804 DEBUG [StoreOpener-3276e02ae096deef8165bc8c50c74666-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:36:48,805 INFO [StoreOpener-3276e02ae096deef8165bc8c50c74666-1 {}] regionserver.HStore(327): Store=3276e02ae096deef8165bc8c50c74666/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:36:48,805 INFO [StoreOpener-3276e02ae096deef8165bc8c50c74666-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3276e02ae096deef8165bc8c50c74666 2024-11-27T21:36:48,806 INFO [StoreOpener-3276e02ae096deef8165bc8c50c74666-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3276e02ae096deef8165bc8c50c74666 columnFamilyName C 2024-11-27T21:36:48,806 DEBUG [StoreOpener-3276e02ae096deef8165bc8c50c74666-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:36:48,806 INFO [StoreOpener-3276e02ae096deef8165bc8c50c74666-1 {}] regionserver.HStore(327): Store=3276e02ae096deef8165bc8c50c74666/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:36:48,806 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1038): replaying wal for 3276e02ae096deef8165bc8c50c74666 2024-11-27T21:36:48,807 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666 2024-11-27T21:36:48,807 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666 2024-11-27T21:36:48,807 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1048): stopping wal replay for 3276e02ae096deef8165bc8c50c74666 2024-11-27T21:36:48,807 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1060): Cleaning up temporary data for 3276e02ae096deef8165bc8c50c74666 2024-11-27T21:36:48,808 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T21:36:48,809 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1093): writing seq id for 3276e02ae096deef8165bc8c50c74666 2024-11-27T21:36:48,810 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T21:36:48,810 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1114): Opened 3276e02ae096deef8165bc8c50c74666; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72207680, jitterRate=0.07597827911376953}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T21:36:48,810 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3276e02ae096deef8165bc8c50c74666 2024-11-27T21:36:48,811 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1006): Region open journal for 3276e02ae096deef8165bc8c50c74666: Running coprocessor pre-open hook at 1732743408801Writing region info on filesystem at 1732743408801Initializing all the Stores at 1732743408801Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732743408801Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732743408802 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732743408802Cleaning up temporary data from old regions at 1732743408807 (+5 ms)Running coprocessor post-open hooks at 1732743408810 (+3 ms)Region opened successfully at 1732743408811 (+1 ms) 2024-11-27T21:36:48,811 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., pid=154, masterSystemTime=1732743408798 2024-11-27T21:36:48,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-27T21:36:48,813 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:48,813 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:48,814 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=3276e02ae096deef8165bc8c50c74666, regionState=OPEN, openSeqNum=2, regionLocation=4585c39bed03,39097,1732743267048 2024-11-27T21:36:48,815 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=154, ppid=153, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 because future has completed 2024-11-27T21:36:48,817 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=154, resume processing ppid=153 2024-11-27T21:36:48,817 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, ppid=153, state=SUCCESS, hasLock=false; OpenRegionProcedure 3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 in 169 msec 2024-11-27T21:36:48,819 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-27T21:36:48,819 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3276e02ae096deef8165bc8c50c74666, ASSIGN in 325 msec 2024-11-27T21:36:48,820 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T21:36:48,820 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732743408820"}]},"ts":"1732743408820"} 2024-11-27T21:36:48,821 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-27T21:36:48,822 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T21:36:48,825 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1500 sec 2024-11-27T21:36:49,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-27T21:36:49,824 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:36:49,825 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1227a368, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:49,825 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:36:49,825 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:36:49,826 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:36:49,826 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:36:49,826 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:36:49,826 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@868134d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:49,826 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:36:49,826 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:36:49,827 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:49,827 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37492, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:36:49,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bc315cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:49,828 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:36:49,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:36:49,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:36:49,830 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52880, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:36:49,830 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T21:36:49,831 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 4585c39bed03,34433,1732743266181 2024-11-27T21:36:49,831 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7e7d04c9 2024-11-27T21:36:49,831 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T21:36:49,832 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37496, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T21:36:49,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T21:36:49,833 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T21:36:49,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=155, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-27T21:36:49,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742390_1566 (size=995) 2024-11-27T21:36:50,244 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-27T21:36:50,244 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-27T21:36:50,246 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=156, ppid=155, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T21:36:50,248 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3276e02ae096deef8165bc8c50c74666, REOPEN/MOVE}] 2024-11-27T21:36:50,248 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3276e02ae096deef8165bc8c50c74666, REOPEN/MOVE 2024-11-27T21:36:50,249 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=157 updating hbase:meta row=3276e02ae096deef8165bc8c50c74666, regionState=CLOSING, regionLocation=4585c39bed03,39097,1732743267048 2024-11-27T21:36:50,250 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3276e02ae096deef8165bc8c50c74666, REOPEN/MOVE because future has completed 2024-11-27T21:36:50,250 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T21:36:50,250 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=158, ppid=157, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048}] 2024-11-27T21:36:50,402 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] handler.UnassignRegionHandler(122): Close 3276e02ae096deef8165bc8c50c74666 2024-11-27T21:36:50,402 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T21:36:50,402 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1722): Closing 3276e02ae096deef8165bc8c50c74666, disabling compactions & flushes 2024-11-27T21:36:50,402 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:50,402 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:50,402 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. after waiting 0 ms 2024-11-27T21:36:50,402 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:50,405 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T21:36:50,406 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:50,406 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1676): Region close journal for 3276e02ae096deef8165bc8c50c74666: Waiting for close lock at 1732743410402Running coprocessor pre-close hooks at 1732743410402Disabling compacts and flushes for region at 1732743410402Disabling writes for close at 1732743410402Writing region close event to WAL at 1732743410403 (+1 ms)Running coprocessor post-close hooks at 1732743410406 (+3 ms)Closed at 1732743410406 2024-11-27T21:36:50,406 WARN [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegionServer(3299): Not adding moved region record: 3276e02ae096deef8165bc8c50c74666 to self. 2024-11-27T21:36:50,407 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] handler.UnassignRegionHandler(157): Closed 3276e02ae096deef8165bc8c50c74666 2024-11-27T21:36:50,408 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=157 updating hbase:meta row=3276e02ae096deef8165bc8c50c74666, regionState=CLOSED 2024-11-27T21:36:50,409 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=158, ppid=157, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 because future has completed 2024-11-27T21:36:50,411 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=158, resume processing ppid=157 2024-11-27T21:36:50,411 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, ppid=157, state=SUCCESS, hasLock=false; CloseRegionProcedure 3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 in 159 msec 2024-11-27T21:36:50,411 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3276e02ae096deef8165bc8c50c74666, REOPEN/MOVE; state=CLOSED, location=4585c39bed03,39097,1732743267048; forceNewPlan=false, retain=true 2024-11-27T21:36:50,562 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=157 updating hbase:meta row=3276e02ae096deef8165bc8c50c74666, regionState=OPENING, regionLocation=4585c39bed03,39097,1732743267048 2024-11-27T21:36:50,563 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3276e02ae096deef8165bc8c50c74666, REOPEN/MOVE because future has completed 2024-11-27T21:36:50,563 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=157, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048}] 2024-11-27T21:36:50,718 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:50,718 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(7752): Opening region: {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} 2024-11-27T21:36:50,718 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3276e02ae096deef8165bc8c50c74666 2024-11-27T21:36:50,718 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T21:36:50,718 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(7794): checking encryption for 3276e02ae096deef8165bc8c50c74666 2024-11-27T21:36:50,718 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(7797): checking classloading for 3276e02ae096deef8165bc8c50c74666 2024-11-27T21:36:50,719 INFO [StoreOpener-3276e02ae096deef8165bc8c50c74666-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3276e02ae096deef8165bc8c50c74666 2024-11-27T21:36:50,720 INFO [StoreOpener-3276e02ae096deef8165bc8c50c74666-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3276e02ae096deef8165bc8c50c74666 columnFamilyName A 2024-11-27T21:36:50,721 DEBUG [StoreOpener-3276e02ae096deef8165bc8c50c74666-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:36:50,721 INFO [StoreOpener-3276e02ae096deef8165bc8c50c74666-1 {}] regionserver.HStore(327): Store=3276e02ae096deef8165bc8c50c74666/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:36:50,721 INFO [StoreOpener-3276e02ae096deef8165bc8c50c74666-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3276e02ae096deef8165bc8c50c74666 2024-11-27T21:36:50,722 INFO [StoreOpener-3276e02ae096deef8165bc8c50c74666-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3276e02ae096deef8165bc8c50c74666 columnFamilyName B 2024-11-27T21:36:50,722 DEBUG [StoreOpener-3276e02ae096deef8165bc8c50c74666-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:36:50,722 INFO [StoreOpener-3276e02ae096deef8165bc8c50c74666-1 {}] regionserver.HStore(327): Store=3276e02ae096deef8165bc8c50c74666/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:36:50,722 INFO [StoreOpener-3276e02ae096deef8165bc8c50c74666-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3276e02ae096deef8165bc8c50c74666 2024-11-27T21:36:50,723 INFO [StoreOpener-3276e02ae096deef8165bc8c50c74666-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3276e02ae096deef8165bc8c50c74666 columnFamilyName C 2024-11-27T21:36:50,723 DEBUG [StoreOpener-3276e02ae096deef8165bc8c50c74666-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:36:50,723 INFO [StoreOpener-3276e02ae096deef8165bc8c50c74666-1 {}] regionserver.HStore(327): Store=3276e02ae096deef8165bc8c50c74666/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:36:50,723 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1038): replaying wal for 3276e02ae096deef8165bc8c50c74666 2024-11-27T21:36:50,724 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666 2024-11-27T21:36:50,725 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666 2024-11-27T21:36:50,725 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1048): stopping wal replay for 3276e02ae096deef8165bc8c50c74666 2024-11-27T21:36:50,725 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1060): Cleaning up temporary data for 3276e02ae096deef8165bc8c50c74666 2024-11-27T21:36:50,726 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T21:36:50,727 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1093): writing seq id for 3276e02ae096deef8165bc8c50c74666 2024-11-27T21:36:50,727 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1114): Opened 3276e02ae096deef8165bc8c50c74666; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66030712, jitterRate=-0.01606571674346924}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T21:36:50,727 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3276e02ae096deef8165bc8c50c74666 2024-11-27T21:36:50,728 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1006): Region open journal for 3276e02ae096deef8165bc8c50c74666: Running coprocessor pre-open hook at 1732743410718Writing region info on filesystem at 1732743410718Initializing all the Stores at 1732743410719 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732743410719Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732743410719Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732743410719Cleaning up temporary data from old regions at 1732743410725 (+6 ms)Running coprocessor post-open hooks at 1732743410728 (+3 ms)Region opened successfully at 1732743410728 2024-11-27T21:36:50,729 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., pid=159, masterSystemTime=1732743410715 2024-11-27T21:36:50,730 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:50,730 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:50,731 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=157 updating hbase:meta row=3276e02ae096deef8165bc8c50c74666, regionState=OPEN, openSeqNum=5, regionLocation=4585c39bed03,39097,1732743267048 2024-11-27T21:36:50,732 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=159, ppid=157, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 because future has completed 2024-11-27T21:36:50,734 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=157 2024-11-27T21:36:50,734 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=157, state=SUCCESS, hasLock=false; OpenRegionProcedure 3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 in 170 msec 2024-11-27T21:36:50,735 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-27T21:36:50,735 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3276e02ae096deef8165bc8c50c74666, REOPEN/MOVE in 486 msec 2024-11-27T21:36:50,738 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=156, resume processing ppid=155 2024-11-27T21:36:50,738 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, ppid=155, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 490 msec 2024-11-27T21:36:50,739 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 904 msec 2024-11-27T21:36:50,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=155 2024-11-27T21:36:50,740 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:36:50,741 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b3732f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:50,741 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:36:50,741 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:36:50,742 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:36:50,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:36:50,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:36:50,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7db76a17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:50,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:36:50,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:36:50,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:50,743 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:36:50,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48e715f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:50,745 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ee989f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:50,745 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:36:50,745 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:36:50,746 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:36:50,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:36:50,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:36:50,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bc158d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:50,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:36:50,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:36:50,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:50,747 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33842, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:36:50,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33d23dfa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:50,748 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41209cef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:50,748 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:36:50,748 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:36:50,749 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:36:50,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:36:50,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:36:50,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f34b90c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:50,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:36:50,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:36:50,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:50,750 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33858, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:36:50,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b7d8411, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:50,751 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74a0408c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:50,751 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:36:50,752 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:36:50,752 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:36:50,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:36:50,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:36:50,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d041311, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:50,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:36:50,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:36:50,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:50,754 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33874, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:36:50,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32564313, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:50,755 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6187a3f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:50,755 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:36:50,755 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:36:50,756 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:36:50,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:36:50,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:36:50,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4271f820, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:50,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:36:50,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:36:50,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:50,757 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33898, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:36:50,757 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1750a105, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:50,758 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1407fbe0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:50,758 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:36:50,758 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:36:50,759 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:36:50,759 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:36:50,759 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:36:50,759 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38b3b036, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:50,759 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:36:50,760 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:36:50,760 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:50,760 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33918, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:36:50,761 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b3782dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:50,761 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b3d6c0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:50,761 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:36:50,761 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:36:50,762 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:36:50,762 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:36:50,762 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:36:50,762 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ab81603, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:50,763 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:36:50,763 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:36:50,763 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:50,764 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33942, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:36:50,764 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bf30e41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:50,765 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@600a9875, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:50,765 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:36:50,765 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:36:50,766 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:36:50,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:36:50,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:36:50,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bc35f83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:50,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:36:50,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:36:50,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:50,767 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33950, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:36:50,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6621ad30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:50,768 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@222a1478, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:50,768 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:36:50,768 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:36:50,769 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:36:50,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:36:50,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:36:50,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e32af6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:50,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:36:50,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:36:50,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:50,770 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33960, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:36:50,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f502e3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:50,771 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9b51f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:50,771 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,34433,-1 for getting cluster id 2024-11-27T21:36:50,771 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:36:50,773 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2' 2024-11-27T21:36:50,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:36:50,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e5c85267-8939-4e31-b5a9-5a3dd6ac9fb2" 2024-11-27T21:36:50,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bef8603, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:50,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,34433,-1] 2024-11-27T21:36:50,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:36:50,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:36:50,774 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33982, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:36:50,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bfcfaa5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:36:50,775 DEBUG [Thread-2519 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:36:50,775 DEBUG [Thread-2512 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:36:50,775 DEBUG [Thread-2521 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:36:50,775 DEBUG [Thread-2506 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:36:50,776 DEBUG [Thread-2510 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:36:50,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:36:50,776 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:36:50,776 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:36:50,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:36:50,776 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:36:50,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:36:50,776 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:36:50,776 DEBUG [Thread-2517 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:36:50,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:36:50,777 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:36:50,777 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45640, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:36:50,777 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:36:50,777 DEBUG [Thread-2515 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:36:50,777 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:36:50,777 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45624, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:36:50,778 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:36:50,778 DEBUG [Thread-2504 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:36:50,778 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:36:50,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:36:50,778 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:36:50,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:36:50,778 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45656, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:36:50,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5] 2024-11-27T21:36:50,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5] 2024-11-27T21:36:50,779 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45660, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:36:50,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5] 2024-11-27T21:36:50,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5] 2024-11-27T21:36:50,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5] 2024-11-27T21:36:50,780 DEBUG [Thread-2523 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:36:50,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5] 2024-11-27T21:36:50,781 DEBUG [Thread-2508 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:36:50,781 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:36:50,781 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:36:50,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:36:50,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:36:50,781 DEBUG [Thread-2508 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,39097,1732743267048, seqNum=-1] 2024-11-27T21:36:50,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:36:50,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:36:50,782 DEBUG [Thread-2508 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:36:50,783 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45678, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:36:50,783 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45672, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:36:50,784 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5] 2024-11-27T21:36:50,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-27T21:36:50,784 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5] 2024-11-27T21:36:50,785 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45682, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:36:50,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5] 2024-11-27T21:36:50,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-27T21:36:50,786 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:36:50,786 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5] 2024-11-27T21:36:50,787 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:36:50,787 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:36:50,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on 3276e02ae096deef8165bc8c50c74666 2024-11-27T21:36:50,800 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:36:50,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3276e02ae096deef8165bc8c50c74666 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-27T21:36:50,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127abf1ef7edad44ea2b19f6db293ed26b5_3276e02ae096deef8165bc8c50c74666 is 50, key is test_row_0/A:col10/1732743410800/Put/seqid=0 2024-11-27T21:36:50,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:50,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45662 deadline: 1732743470830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:50,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:50,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:50,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:50,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:50,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743470831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:50,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:50,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:50,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:50,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743470831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:50,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:50,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:50,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743470833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:50,836 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:50,836 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:50,836 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:50,836 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:50,836 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:50,836 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:50,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:50,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732743470834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:50,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:50,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:50,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:50,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742391_1567 (size=14594) 2024-11-27T21:36:50,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-27T21:36:50,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T21:36:50,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:50,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. as already flushing 2024-11-27T21:36:50,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:50,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:50,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:50,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:50,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:50,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743470944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:50,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:50,947 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:50,947 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:50,947 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:50,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45662 deadline: 1732743470944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:50,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:50,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:50,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:50,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:50,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:50,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732743470944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:50,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743470944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:50,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:50,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:50,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:50,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743470944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:50,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:50,948 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:50,948 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:50,948 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:50,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:50,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:50,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:51,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T21:36:51,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:51,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. as already flushing 2024-11-27T21:36:51,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:51,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:51,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:51,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:51,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-27T21:36:51,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:51,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743471154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:51,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:51,156 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:51,156 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:51,156 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:51,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45662 deadline: 1732743471154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:51,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:51,156 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:51,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732743471154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:51,156 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:51,156 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:51,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:51,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:51,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:51,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:51,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743471154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:51,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:51,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:51,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743471154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:51,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:51,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:51,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:51,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:51,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:51,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T21:36:51,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:51,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. as already flushing 2024-11-27T21:36:51,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:51,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:51,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:51,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:51,249 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:36:51,255 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127abf1ef7edad44ea2b19f6db293ed26b5_3276e02ae096deef8165bc8c50c74666 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127abf1ef7edad44ea2b19f6db293ed26b5_3276e02ae096deef8165bc8c50c74666 2024-11-27T21:36:51,256 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/a848f715f9ba43bc8c1c545baa1b3090, store: [table=TestAcidGuarantees family=A region=3276e02ae096deef8165bc8c50c74666] 2024-11-27T21:36:51,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/a848f715f9ba43bc8c1c545baa1b3090 is 175, key is test_row_0/A:col10/1732743410800/Put/seqid=0 2024-11-27T21:36:51,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742392_1568 (size=39549) 2024-11-27T21:36:51,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T21:36:51,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:51,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. as already flushing 2024-11-27T21:36:51,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:51,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:51,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:51,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:51,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-27T21:36:51,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743471464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:51,466 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:51,466 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:51,466 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:51,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:51,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45662 deadline: 1732743471464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:51,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:51,467 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:51,467 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:51,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732743471464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:51,467 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:51,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:51,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:51,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743471464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:51,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:51,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:51,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:51,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:51,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743471464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:51,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:51,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:51,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:51,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:51,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:51,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T21:36:51,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:51,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. as already flushing 2024-11-27T21:36:51,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:51,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:51,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:51,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:51,661 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=18, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/a848f715f9ba43bc8c1c545baa1b3090 2024-11-27T21:36:51,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/2b1051f5e0584c9fa332ab5591a6ba0f is 50, key is test_row_0/B:col10/1732743410800/Put/seqid=0 2024-11-27T21:36:51,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742393_1569 (size=12001) 2024-11-27T21:36:51,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T21:36:51,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:51,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. as already flushing 2024-11-27T21:36:51,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:51,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:51,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:51,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T21:36:51,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:51,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. as already flushing 2024-11-27T21:36:51,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:51,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:51,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:51,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:51,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-27T21:36:51,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:51,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743471973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:51,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:51,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45662 deadline: 1732743471974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:51,977 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:51,977 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:51,977 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:51,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:51,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:51,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:51,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732743471974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:51,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:51,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:51,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:51,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743471974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:51,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:51,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:51,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:51,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743471974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:51,977 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:51,978 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:51,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:51,978 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:51,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:51,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:52,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T21:36:52,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:52,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. as already flushing 2024-11-27T21:36:52,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:52,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:52,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:52,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:52,086 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/2b1051f5e0584c9fa332ab5591a6ba0f 2024-11-27T21:36:52,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/ce92d251a1024273a919370a09aa4368 is 50, key is test_row_0/C:col10/1732743410800/Put/seqid=0 2024-11-27T21:36:52,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742394_1570 (size=12001) 2024-11-27T21:36:52,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T21:36:52,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:52,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. as already flushing 2024-11-27T21:36:52,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:52,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:52,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:52,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:52,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T21:36:52,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:52,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. as already flushing 2024-11-27T21:36:52,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:52,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:52,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:52,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:52,384 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T21:36:52,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T21:36:52,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:52,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. as already flushing 2024-11-27T21:36:52,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:52,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:52,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:52,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:52,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/ce92d251a1024273a919370a09aa4368 2024-11-27T21:36:52,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/a848f715f9ba43bc8c1c545baa1b3090 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/a848f715f9ba43bc8c1c545baa1b3090 2024-11-27T21:36:52,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/a848f715f9ba43bc8c1c545baa1b3090, entries=200, sequenceid=18, filesize=38.6 K 2024-11-27T21:36:52,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/2b1051f5e0584c9fa332ab5591a6ba0f as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/2b1051f5e0584c9fa332ab5591a6ba0f 2024-11-27T21:36:52,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/2b1051f5e0584c9fa332ab5591a6ba0f, entries=150, sequenceid=18, filesize=11.7 K 2024-11-27T21:36:52,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/ce92d251a1024273a919370a09aa4368 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/ce92d251a1024273a919370a09aa4368 2024-11-27T21:36:52,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/ce92d251a1024273a919370a09aa4368, entries=150, sequenceid=18, filesize=11.7 K 2024-11-27T21:36:52,530 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 3276e02ae096deef8165bc8c50c74666 in 1730ms, sequenceid=18, compaction requested=false 2024-11-27T21:36:52,530 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-27T21:36:52,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3276e02ae096deef8165bc8c50c74666: 2024-11-27T21:36:52,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T21:36:52,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:52,623 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 3276e02ae096deef8165bc8c50c74666 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-27T21:36:52,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411273932d16fd690413887eef2f257577747_3276e02ae096deef8165bc8c50c74666 is 50, key is test_row_0/A:col10/1732743410830/Put/seqid=0 2024-11-27T21:36:52,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742395_1571 (size=12154) 2024-11-27T21:36:52,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-27T21:36:52,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on 3276e02ae096deef8165bc8c50c74666 2024-11-27T21:36:52,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. as already flushing 2024-11-27T21:36:53,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:53,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743472999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:53,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:53,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:53,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:53,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:53,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743473002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:53,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:53,007 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:53,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743473002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:53,007 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:53,007 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:53,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:53,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732743473003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:53,007 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:53,007 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:53,007 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:53,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:53,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:53,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45662 deadline: 1732743473003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:53,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:53,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:53,007 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:53,007 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:53,007 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:53,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:36:53,045 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411273932d16fd690413887eef2f257577747_3276e02ae096deef8165bc8c50c74666 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411273932d16fd690413887eef2f257577747_3276e02ae096deef8165bc8c50c74666 2024-11-27T21:36:53,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/33204ce5f10b4e729cace3f02e00dbc9, store: [table=TestAcidGuarantees family=A region=3276e02ae096deef8165bc8c50c74666] 2024-11-27T21:36:53,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/33204ce5f10b4e729cace3f02e00dbc9 is 175, key is test_row_0/A:col10/1732743410830/Put/seqid=0 2024-11-27T21:36:53,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742396_1572 (size=30955) 2024-11-27T21:36:53,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:53,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743473113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:53,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:53,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:53,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743473113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:53,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:53,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:53,116 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:53,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:53,116 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:53,116 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:53,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732743473113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:53,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:53,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:53,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:53,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:53,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45662 deadline: 1732743473113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:53,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:53,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743473113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:53,117 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:53,117 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:53,117 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:53,117 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:53,117 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:53,117 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:53,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:53,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743473323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:53,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:53,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:53,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:53,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743473323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:53,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:53,326 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:53,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:53,326 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:53,326 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:53,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732743473323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:53,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:53,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:53,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:53,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:53,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45662 deadline: 1732743473323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:53,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:53,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:53,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:53,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:53,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743473323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:53,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:53,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:53,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:53,452 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/33204ce5f10b4e729cace3f02e00dbc9 2024-11-27T21:36:53,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/a9f94b1a7b254d7193d42e69895a02e2 is 50, key is test_row_0/B:col10/1732743410830/Put/seqid=0 2024-11-27T21:36:53,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742397_1573 (size=12001) 2024-11-27T21:36:53,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:53,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743473634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:53,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:53,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:53,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:53,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732743473634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:53,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:53,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:53,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:53,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:53,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:53,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743473634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:53,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:53,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743473634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:53,637 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:53,637 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:53,637 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:53,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:53,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:53,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:53,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:53,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45662 deadline: 1732743473634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:53,638 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:53,638 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:53,638 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:53,861 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/a9f94b1a7b254d7193d42e69895a02e2 2024-11-27T21:36:53,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/0602fb9f9ed24086beb1034da1761bcb is 50, key is test_row_0/C:col10/1732743410830/Put/seqid=0 2024-11-27T21:36:53,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742398_1574 (size=12001) 2024-11-27T21:36:53,872 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/0602fb9f9ed24086beb1034da1761bcb 2024-11-27T21:36:53,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/33204ce5f10b4e729cace3f02e00dbc9 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/33204ce5f10b4e729cace3f02e00dbc9 2024-11-27T21:36:53,879 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/33204ce5f10b4e729cace3f02e00dbc9, entries=150, sequenceid=40, filesize=30.2 K 2024-11-27T21:36:53,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/a9f94b1a7b254d7193d42e69895a02e2 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/a9f94b1a7b254d7193d42e69895a02e2 2024-11-27T21:36:53,891 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/a9f94b1a7b254d7193d42e69895a02e2, entries=150, sequenceid=40, filesize=11.7 K 2024-11-27T21:36:53,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/0602fb9f9ed24086beb1034da1761bcb as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/0602fb9f9ed24086beb1034da1761bcb 2024-11-27T21:36:53,895 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/0602fb9f9ed24086beb1034da1761bcb, entries=150, sequenceid=40, filesize=11.7 K 2024-11-27T21:36:53,896 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for 3276e02ae096deef8165bc8c50c74666 in 1273ms, sequenceid=40, compaction requested=false 2024-11-27T21:36:53,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 3276e02ae096deef8165bc8c50c74666: 2024-11-27T21:36:53,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:53,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-27T21:36:53,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-27T21:36:53,900 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-27T21:36:53,900 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1100 sec 2024-11-27T21:36:53,903 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 3.1200 sec 2024-11-27T21:36:54,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on 3276e02ae096deef8165bc8c50c74666 2024-11-27T21:36:54,145 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:36:54,145 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3276e02ae096deef8165bc8c50c74666 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-27T21:36:54,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127e95e11d22565495c8ad351173e573326_3276e02ae096deef8165bc8c50c74666 is 50, key is test_row_0/A:col10/1732743414145/Put/seqid=0 2024-11-27T21:36:54,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742399_1575 (size=12154) 2024-11-27T21:36:54,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:54,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:54,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45662 deadline: 1732743474168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:54,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743474169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:54,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:54,175 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:54,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:54,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:54,175 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:54,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:54,175 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:54,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743474169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:54,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:54,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:54,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:54,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:54,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732743474170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:54,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:54,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:54,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:54,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:54,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743474170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:54,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:54,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:54,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:54,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:54,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45662 deadline: 1732743474283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:54,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:54,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743474284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:54,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:54,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:54,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:54,286 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:54,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:54,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743474284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:54,286 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:54,286 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:54,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:54,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:54,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:54,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:54,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732743474284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:54,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:54,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:54,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:54,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:54,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743474284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:54,287 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:54,287 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:54,287 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:54,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:54,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45662 deadline: 1732743474493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:54,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:54,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743474493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:54,496 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:54,496 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:54,496 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:54,496 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:54,496 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:54,496 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:54,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:54,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743474493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:54,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:54,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:54,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:54,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732743474493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:54,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:54,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:54,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:54,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:54,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743474493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:54,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:54,497 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:54,497 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:54,497 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:54,557 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:36:54,560 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127e95e11d22565495c8ad351173e573326_3276e02ae096deef8165bc8c50c74666 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127e95e11d22565495c8ad351173e573326_3276e02ae096deef8165bc8c50c74666 2024-11-27T21:36:54,561 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/1655c04c09404ce5923f6ca7d56a2f33, store: [table=TestAcidGuarantees family=A region=3276e02ae096deef8165bc8c50c74666] 2024-11-27T21:36:54,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/1655c04c09404ce5923f6ca7d56a2f33 is 175, key is test_row_0/A:col10/1732743414145/Put/seqid=0 2024-11-27T21:36:54,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742400_1576 (size=30955) 2024-11-27T21:36:54,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:54,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45662 deadline: 1732743474808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:54,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:54,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743474808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:54,810 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:54,810 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:54,810 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:54,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:54,810 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:54,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743474808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:54,810 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:54,810 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:54,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:54,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:54,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:54,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:54,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732743474808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:54,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:54,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:54,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:54,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743474808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:54,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:54,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:54,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:54,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-27T21:36:54,954 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:36:54,954 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:36:54,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-27T21:36:54,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-27T21:36:54,957 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:36:54,958 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:36:54,958 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:36:54,966 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=59, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/1655c04c09404ce5923f6ca7d56a2f33 2024-11-27T21:36:54,972 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/08551a63a5ab4e369ab54a5bdd1c93b8 is 50, key is test_row_0/B:col10/1732743414145/Put/seqid=0 2024-11-27T21:36:54,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742401_1577 (size=12001) 2024-11-27T21:36:55,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-27T21:36:55,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-27T21:36:55,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:55,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. as already flushing 2024-11-27T21:36:55,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:55,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:55,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:55,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:55,212 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T21:36:55,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-27T21:36:55,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:55,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. as already flushing 2024-11-27T21:36:55,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:55,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:55,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:55,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:55,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-27T21:36:55,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:55,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45662 deadline: 1732743475313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:55,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:55,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:55,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:55,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:55,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743475324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:55,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:55,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743475324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:55,326 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:55,327 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:55,327 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:55,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:55,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:55,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:55,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732743475324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:55,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:55,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:55,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:55,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:55,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:55,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743475324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:55,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:55,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:55,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:55,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/08551a63a5ab4e369ab54a5bdd1c93b8 2024-11-27T21:36:55,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/589ecfc0da9044699de21aa620b178c9 is 50, key is test_row_0/C:col10/1732743414145/Put/seqid=0 2024-11-27T21:36:55,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742402_1578 (size=12001) 2024-11-27T21:36:55,387 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/589ecfc0da9044699de21aa620b178c9 2024-11-27T21:36:55,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/1655c04c09404ce5923f6ca7d56a2f33 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/1655c04c09404ce5923f6ca7d56a2f33 2024-11-27T21:36:55,393 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/1655c04c09404ce5923f6ca7d56a2f33, entries=150, sequenceid=59, filesize=30.2 K 2024-11-27T21:36:55,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/08551a63a5ab4e369ab54a5bdd1c93b8 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/08551a63a5ab4e369ab54a5bdd1c93b8 2024-11-27T21:36:55,397 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/08551a63a5ab4e369ab54a5bdd1c93b8, entries=150, sequenceid=59, filesize=11.7 K 2024-11-27T21:36:55,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/589ecfc0da9044699de21aa620b178c9 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/589ecfc0da9044699de21aa620b178c9 2024-11-27T21:36:55,401 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/589ecfc0da9044699de21aa620b178c9, entries=150, sequenceid=59, filesize=11.7 K 2024-11-27T21:36:55,402 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for 3276e02ae096deef8165bc8c50c74666 in 1257ms, sequenceid=59, compaction requested=true 2024-11-27T21:36:55,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3276e02ae096deef8165bc8c50c74666: 2024-11-27T21:36:55,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3276e02ae096deef8165bc8c50c74666:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:36:55,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:55,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3276e02ae096deef8165bc8c50c74666:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:36:55,402 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:55,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:55,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3276e02ae096deef8165bc8c50c74666:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:36:55,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:36:55,402 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:55,403 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:55,403 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:55,403 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): 3276e02ae096deef8165bc8c50c74666/B is initiating minor compaction (all files) 2024-11-27T21:36:55,403 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): 3276e02ae096deef8165bc8c50c74666/A is initiating minor compaction (all files) 2024-11-27T21:36:55,403 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3276e02ae096deef8165bc8c50c74666/B in TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:55,403 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3276e02ae096deef8165bc8c50c74666/A in TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:55,403 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/2b1051f5e0584c9fa332ab5591a6ba0f, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/a9f94b1a7b254d7193d42e69895a02e2, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/08551a63a5ab4e369ab54a5bdd1c93b8] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp, totalSize=35.2 K 2024-11-27T21:36:55,403 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/a848f715f9ba43bc8c1c545baa1b3090, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/33204ce5f10b4e729cace3f02e00dbc9, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/1655c04c09404ce5923f6ca7d56a2f33] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp, totalSize=99.1 K 2024-11-27T21:36:55,403 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:55,403 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. files: [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/a848f715f9ba43bc8c1c545baa1b3090, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/33204ce5f10b4e729cace3f02e00dbc9, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/1655c04c09404ce5923f6ca7d56a2f33] 2024-11-27T21:36:55,403 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b1051f5e0584c9fa332ab5591a6ba0f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1732743410799 2024-11-27T21:36:55,404 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting a848f715f9ba43bc8c1c545baa1b3090, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1732743410799 2024-11-27T21:36:55,404 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting a9f94b1a7b254d7193d42e69895a02e2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732743410830 2024-11-27T21:36:55,404 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33204ce5f10b4e729cace3f02e00dbc9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732743410830 2024-11-27T21:36:55,404 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1655c04c09404ce5923f6ca7d56a2f33, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732743414145 2024-11-27T21:36:55,404 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 08551a63a5ab4e369ab54a5bdd1c93b8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732743414145 2024-11-27T21:36:55,410 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3276e02ae096deef8165bc8c50c74666#B#compaction#498 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:55,411 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/03cd6377235a457e8bc7ae1ec5b0d713 is 50, key is test_row_0/B:col10/1732743414145/Put/seqid=0 2024-11-27T21:36:55,412 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3276e02ae096deef8165bc8c50c74666] 2024-11-27T21:36:55,414 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112778a23e6ff7924a67be8c23c6ff1e2afb_3276e02ae096deef8165bc8c50c74666 store=[table=TestAcidGuarantees family=A region=3276e02ae096deef8165bc8c50c74666] 2024-11-27T21:36:55,416 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112778a23e6ff7924a67be8c23c6ff1e2afb_3276e02ae096deef8165bc8c50c74666, store=[table=TestAcidGuarantees family=A region=3276e02ae096deef8165bc8c50c74666] 2024-11-27T21:36:55,416 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112778a23e6ff7924a67be8c23c6ff1e2afb_3276e02ae096deef8165bc8c50c74666 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3276e02ae096deef8165bc8c50c74666] 2024-11-27T21:36:55,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742403_1579 (size=12104) 2024-11-27T21:36:55,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-27T21:36:55,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:55,419 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 3276e02ae096deef8165bc8c50c74666 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-27T21:36:55,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742404_1580 (size=4469) 2024-11-27T21:36:55,424 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3276e02ae096deef8165bc8c50c74666#A#compaction#499 average throughput is 2.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:55,425 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/b0bca8d6fac047e08f9841e426a8db46 is 175, key is test_row_0/A:col10/1732743414145/Put/seqid=0 2024-11-27T21:36:55,425 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/03cd6377235a457e8bc7ae1ec5b0d713 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/03cd6377235a457e8bc7ae1ec5b0d713 2024-11-27T21:36:55,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d64f160ed82b415986f6408abce62033_3276e02ae096deef8165bc8c50c74666 is 50, key is test_row_0/A:col10/1732743414166/Put/seqid=0 2024-11-27T21:36:55,430 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3276e02ae096deef8165bc8c50c74666/B of 3276e02ae096deef8165bc8c50c74666 into 03cd6377235a457e8bc7ae1ec5b0d713(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:55,430 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3276e02ae096deef8165bc8c50c74666: 2024-11-27T21:36:55,431 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., storeName=3276e02ae096deef8165bc8c50c74666/B, priority=13, startTime=1732743415402; duration=0sec 2024-11-27T21:36:55,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742406_1582 (size=12154) 2024-11-27T21:36:55,431 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:36:55,431 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3276e02ae096deef8165bc8c50c74666:B 2024-11-27T21:36:55,431 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:55,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742405_1581 (size=31058) 2024-11-27T21:36:55,431 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:55,431 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): 3276e02ae096deef8165bc8c50c74666/C is initiating minor compaction (all files) 2024-11-27T21:36:55,432 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3276e02ae096deef8165bc8c50c74666/C in TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:55,432 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/ce92d251a1024273a919370a09aa4368, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/0602fb9f9ed24086beb1034da1761bcb, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/589ecfc0da9044699de21aa620b178c9] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp, totalSize=35.2 K 2024-11-27T21:36:55,432 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting ce92d251a1024273a919370a09aa4368, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1732743410799 2024-11-27T21:36:55,432 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 0602fb9f9ed24086beb1034da1761bcb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732743410830 2024-11-27T21:36:55,433 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 589ecfc0da9044699de21aa620b178c9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732743414145 2024-11-27T21:36:55,438 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3276e02ae096deef8165bc8c50c74666#C#compaction#501 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:55,438 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/76d836b64de84593bb94b1ab631b84bb is 50, key is test_row_0/C:col10/1732743414145/Put/seqid=0 2024-11-27T21:36:55,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742407_1583 (size=12104) 2024-11-27T21:36:55,455 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/76d836b64de84593bb94b1ab631b84bb as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/76d836b64de84593bb94b1ab631b84bb 2024-11-27T21:36:55,459 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3276e02ae096deef8165bc8c50c74666/C of 3276e02ae096deef8165bc8c50c74666 into 76d836b64de84593bb94b1ab631b84bb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:55,459 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3276e02ae096deef8165bc8c50c74666: 2024-11-27T21:36:55,459 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., storeName=3276e02ae096deef8165bc8c50c74666/C, priority=13, startTime=1732743415402; duration=0sec 2024-11-27T21:36:55,459 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:55,459 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3276e02ae096deef8165bc8c50c74666:C 2024-11-27T21:36:55,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-27T21:36:55,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:36:55,838 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d64f160ed82b415986f6408abce62033_3276e02ae096deef8165bc8c50c74666 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d64f160ed82b415986f6408abce62033_3276e02ae096deef8165bc8c50c74666 2024-11-27T21:36:55,840 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/b0bca8d6fac047e08f9841e426a8db46 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/b0bca8d6fac047e08f9841e426a8db46 2024-11-27T21:36:55,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/978b57872d1c4a048bc742182c2059dc, store: [table=TestAcidGuarantees family=A region=3276e02ae096deef8165bc8c50c74666] 2024-11-27T21:36:55,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/978b57872d1c4a048bc742182c2059dc is 175, key is test_row_0/A:col10/1732743414166/Put/seqid=0 2024-11-27T21:36:55,846 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3276e02ae096deef8165bc8c50c74666/A of 3276e02ae096deef8165bc8c50c74666 into b0bca8d6fac047e08f9841e426a8db46(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:55,846 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3276e02ae096deef8165bc8c50c74666: 2024-11-27T21:36:55,846 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., storeName=3276e02ae096deef8165bc8c50c74666/A, priority=13, startTime=1732743415402; duration=0sec 2024-11-27T21:36:55,846 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:55,846 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3276e02ae096deef8165bc8c50c74666:A 2024-11-27T21:36:55,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742408_1584 (size=30955) 2024-11-27T21:36:56,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-27T21:36:56,263 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/978b57872d1c4a048bc742182c2059dc 2024-11-27T21:36:56,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/eac14c90efa2456ea60ae4db5c4c6006 is 50, key is test_row_0/B:col10/1732743414166/Put/seqid=0 2024-11-27T21:36:56,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742409_1585 (size=12001) 2024-11-27T21:36:56,273 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/eac14c90efa2456ea60ae4db5c4c6006 2024-11-27T21:36:56,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/4fcb4f3258614156bd6d6f31a10422b7 is 50, key is test_row_0/C:col10/1732743414166/Put/seqid=0 2024-11-27T21:36:56,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742410_1586 (size=12001) 2024-11-27T21:36:56,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. as already flushing 2024-11-27T21:36:56,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on 3276e02ae096deef8165bc8c50c74666 2024-11-27T21:36:56,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:56,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743476348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:56,355 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:56,355 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:56,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:56,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45662 deadline: 1732743476352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:56,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:56,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:56,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743476353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:56,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732743476354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:56,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:56,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:56,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743476355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:56,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:56,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:56,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:56,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:56,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:56,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:56,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:56,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:56,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:56,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743476463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:56,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:56,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:56,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:56,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45662 deadline: 1732743476463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:56,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:56,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:56,467 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:56,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743476463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:56,467 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:56,467 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:56,467 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:56,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:56,467 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:56,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:56,467 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:56,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732743476463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:56,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743476464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:56,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:56,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:56,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:56,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:56,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:56,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:56,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:56,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743476673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:56,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:56,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:56,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:56,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:56,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45662 deadline: 1732743476673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:56,676 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:56,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:56,676 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:56,676 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:56,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743476674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:56,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:56,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732743476674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:56,677 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:56,677 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:56,677 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:56,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:56,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:56,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:56,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:56,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743476674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:56,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:56,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:56,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:56,683 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/4fcb4f3258614156bd6d6f31a10422b7 2024-11-27T21:36:56,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/978b57872d1c4a048bc742182c2059dc as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/978b57872d1c4a048bc742182c2059dc 2024-11-27T21:36:56,690 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/978b57872d1c4a048bc742182c2059dc, entries=150, sequenceid=77, filesize=30.2 K 2024-11-27T21:36:56,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/eac14c90efa2456ea60ae4db5c4c6006 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/eac14c90efa2456ea60ae4db5c4c6006 2024-11-27T21:36:56,694 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/eac14c90efa2456ea60ae4db5c4c6006, entries=150, sequenceid=77, filesize=11.7 K 2024-11-27T21:36:56,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/4fcb4f3258614156bd6d6f31a10422b7 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/4fcb4f3258614156bd6d6f31a10422b7 2024-11-27T21:36:56,698 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/4fcb4f3258614156bd6d6f31a10422b7, entries=150, sequenceid=77, filesize=11.7 K 2024-11-27T21:36:56,699 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 3276e02ae096deef8165bc8c50c74666 in 1281ms, sequenceid=77, compaction requested=false 2024-11-27T21:36:56,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 3276e02ae096deef8165bc8c50c74666: 2024-11-27T21:36:56,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:56,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-27T21:36:56,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-27T21:36:56,702 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-27T21:36:56,702 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7420 sec 2024-11-27T21:36:56,703 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.7470 sec 2024-11-27T21:36:56,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on 3276e02ae096deef8165bc8c50c74666 2024-11-27T21:36:56,986 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:36:56,986 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3276e02ae096deef8165bc8c50c74666 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-27T21:36:56,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127e5753fa982234300b8d027bbe5ff89c7_3276e02ae096deef8165bc8c50c74666 is 50, key is test_row_0/A:col10/1732743416354/Put/seqid=0 2024-11-27T21:36:56,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742411_1587 (size=12154) 2024-11-27T21:36:56,998 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:36:57,000 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127e5753fa982234300b8d027bbe5ff89c7_3276e02ae096deef8165bc8c50c74666 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127e5753fa982234300b8d027bbe5ff89c7_3276e02ae096deef8165bc8c50c74666 2024-11-27T21:36:57,001 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/a1d3edb097024ad9a07896dfe38f99d2, store: [table=TestAcidGuarantees family=A region=3276e02ae096deef8165bc8c50c74666] 2024-11-27T21:36:57,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/a1d3edb097024ad9a07896dfe38f99d2 is 175, key is test_row_0/A:col10/1732743416354/Put/seqid=0 2024-11-27T21:36:57,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:57,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743477000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:57,003 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:57,003 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:57,003 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:57,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:57,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45662 deadline: 1732743477002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:57,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:57,007 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:57,007 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:57,007 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:57,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743477002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:57,007 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:57,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:57,007 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:57,007 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:57,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:57,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732743477003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:57,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743477004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:57,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:57,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:57,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:57,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:57,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:57,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:57,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742412_1588 (size=30955) 2024-11-27T21:36:57,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-27T21:36:57,113 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:36:57,114 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:36:57,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-27T21:36:57,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:57,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743477113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:57,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:57,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45662 deadline: 1732743477113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:57,115 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:57,115 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:57,115 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:57,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:57,115 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:57,115 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:57,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743477113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:57,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-27T21:36:57,115 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:57,116 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:36:57,116 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:57,116 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:57,116 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:57,116 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:36:57,117 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:36:57,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:57,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732743477113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:57,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:57,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:57,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:57,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743477113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:57,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:57,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:57,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:57,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:57,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-27T21:36:57,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-27T21:36:57,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:57,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. as already flushing 2024-11-27T21:36:57,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:57,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:57,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:57,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743477323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:57,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45662 deadline: 1732743477323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:57,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743477324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:57,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732743477324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:57,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743477324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:57,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:57,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:57,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:57,413 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=101, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/a1d3edb097024ad9a07896dfe38f99d2 2024-11-27T21:36:57,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-27T21:36:57,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/e6b4c7a470ee4c05a70c0b5b7b8597e3 is 50, key is test_row_0/B:col10/1732743416354/Put/seqid=0 2024-11-27T21:36:57,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:57,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. as already flushing 2024-11-27T21:36:57,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:57,422 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:57,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:57,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:57,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-27T21:36:57,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742413_1589 (size=12001) 2024-11-27T21:36:57,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-27T21:36:57,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:57,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. as already flushing 2024-11-27T21:36:57,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:57,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:57,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:57,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:57,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:57,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743477633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:57,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:57,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:57,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45662 deadline: 1732743477633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:57,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:57,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:57,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:57,637 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:57,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743477633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:57,637 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:57,638 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:57,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:57,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732743477634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:57,638 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:57,638 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:57,638 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:57,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:57,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:57,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:57,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:57,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743477634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:57,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:57,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:57,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:57,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-27T21:36:57,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:57,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. as already flushing 2024-11-27T21:36:57,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:57,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:57,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:57,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:57,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-27T21:36:57,843 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/e6b4c7a470ee4c05a70c0b5b7b8597e3 2024-11-27T21:36:57,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/1cdae9fe5ac84b0ea9b391f0b141e631 is 50, key is test_row_0/C:col10/1732743416354/Put/seqid=0 2024-11-27T21:36:57,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742414_1590 (size=12001) 2024-11-27T21:36:57,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-27T21:36:57,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:57,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. as already flushing 2024-11-27T21:36:57,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:57,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:57,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:57,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:58,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-27T21:36:58,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:58,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. as already flushing 2024-11-27T21:36:58,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:58,042 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:58,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:58,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:58,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:58,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743478143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:58,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:58,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45662 deadline: 1732743478143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:58,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:58,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:58,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:58,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:58,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:58,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:58,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743478143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:58,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:58,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:58,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:58,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:58,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:58,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743478153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:58,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:58,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732743478153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:58,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:58,159 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:58,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:58,159 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:58,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:58,159 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:58,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-27T21:36:58,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:58,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. as already flushing 2024-11-27T21:36:58,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:58,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:58,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:58,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:36:58,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-27T21:36:58,258 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/1cdae9fe5ac84b0ea9b391f0b141e631 2024-11-27T21:36:58,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/a1d3edb097024ad9a07896dfe38f99d2 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/a1d3edb097024ad9a07896dfe38f99d2 2024-11-27T21:36:58,265 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/a1d3edb097024ad9a07896dfe38f99d2, entries=150, sequenceid=101, filesize=30.2 K 2024-11-27T21:36:58,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/e6b4c7a470ee4c05a70c0b5b7b8597e3 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/e6b4c7a470ee4c05a70c0b5b7b8597e3 2024-11-27T21:36:58,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/e6b4c7a470ee4c05a70c0b5b7b8597e3, entries=150, sequenceid=101, filesize=11.7 K 2024-11-27T21:36:58,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/1cdae9fe5ac84b0ea9b391f0b141e631 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/1cdae9fe5ac84b0ea9b391f0b141e631 2024-11-27T21:36:58,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/1cdae9fe5ac84b0ea9b391f0b141e631, entries=150, sequenceid=101, filesize=11.7 K 2024-11-27T21:36:58,276 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 3276e02ae096deef8165bc8c50c74666 in 1290ms, sequenceid=101, compaction requested=true 2024-11-27T21:36:58,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3276e02ae096deef8165bc8c50c74666: 2024-11-27T21:36:58,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3276e02ae096deef8165bc8c50c74666:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:36:58,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:58,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3276e02ae096deef8165bc8c50c74666:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:36:58,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:58,276 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:58,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3276e02ae096deef8165bc8c50c74666:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:36:58,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:36:58,276 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:58,277 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:58,277 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:58,277 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): 3276e02ae096deef8165bc8c50c74666/A is initiating minor compaction (all files) 2024-11-27T21:36:58,277 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): 3276e02ae096deef8165bc8c50c74666/B is initiating minor compaction (all files) 2024-11-27T21:36:58,277 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3276e02ae096deef8165bc8c50c74666/A in TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:58,277 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3276e02ae096deef8165bc8c50c74666/B in TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:58,277 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/b0bca8d6fac047e08f9841e426a8db46, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/978b57872d1c4a048bc742182c2059dc, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/a1d3edb097024ad9a07896dfe38f99d2] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp, totalSize=90.8 K 2024-11-27T21:36:58,277 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:58,277 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/03cd6377235a457e8bc7ae1ec5b0d713, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/eac14c90efa2456ea60ae4db5c4c6006, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/e6b4c7a470ee4c05a70c0b5b7b8597e3] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp, totalSize=35.3 K 2024-11-27T21:36:58,277 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. files: [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/b0bca8d6fac047e08f9841e426a8db46, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/978b57872d1c4a048bc742182c2059dc, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/a1d3edb097024ad9a07896dfe38f99d2] 2024-11-27T21:36:58,277 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 03cd6377235a457e8bc7ae1ec5b0d713, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732743414145 2024-11-27T21:36:58,277 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0bca8d6fac047e08f9841e426a8db46, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732743414145 2024-11-27T21:36:58,278 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 978b57872d1c4a048bc742182c2059dc, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732743414166 2024-11-27T21:36:58,278 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting eac14c90efa2456ea60ae4db5c4c6006, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732743414166 2024-11-27T21:36:58,278 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1d3edb097024ad9a07896dfe38f99d2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1732743416354 2024-11-27T21:36:58,278 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting e6b4c7a470ee4c05a70c0b5b7b8597e3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1732743416354 2024-11-27T21:36:58,284 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3276e02ae096deef8165bc8c50c74666] 2024-11-27T21:36:58,286 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3276e02ae096deef8165bc8c50c74666#B#compaction#508 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:58,286 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/d294f9c02a3f4db38d8ae9e19af5357b is 50, key is test_row_0/B:col10/1732743416354/Put/seqid=0 2024-11-27T21:36:58,288 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127be60049149ed4efd8b166f0f4c868d43_3276e02ae096deef8165bc8c50c74666 store=[table=TestAcidGuarantees family=A region=3276e02ae096deef8165bc8c50c74666] 2024-11-27T21:36:58,289 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127be60049149ed4efd8b166f0f4c868d43_3276e02ae096deef8165bc8c50c74666, store=[table=TestAcidGuarantees family=A region=3276e02ae096deef8165bc8c50c74666] 2024-11-27T21:36:58,289 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127be60049149ed4efd8b166f0f4c868d43_3276e02ae096deef8165bc8c50c74666 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3276e02ae096deef8165bc8c50c74666] 2024-11-27T21:36:58,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742415_1591 (size=12207) 2024-11-27T21:36:58,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742416_1592 (size=4469) 2024-11-27T21:36:58,307 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3276e02ae096deef8165bc8c50c74666#A#compaction#507 average throughput is 1.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:58,307 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/993fe8e5e46a4022aca3a9fc28f6cdf1 is 175, key is test_row_0/A:col10/1732743416354/Put/seqid=0 2024-11-27T21:36:58,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742417_1593 (size=31161) 2024-11-27T21:36:58,315 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/993fe8e5e46a4022aca3a9fc28f6cdf1 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/993fe8e5e46a4022aca3a9fc28f6cdf1 2024-11-27T21:36:58,320 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3276e02ae096deef8165bc8c50c74666/A of 3276e02ae096deef8165bc8c50c74666 into 993fe8e5e46a4022aca3a9fc28f6cdf1(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:58,320 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3276e02ae096deef8165bc8c50c74666: 2024-11-27T21:36:58,321 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., storeName=3276e02ae096deef8165bc8c50c74666/A, priority=13, startTime=1732743418276; duration=0sec 2024-11-27T21:36:58,321 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:36:58,321 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3276e02ae096deef8165bc8c50c74666:A 2024-11-27T21:36:58,321 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:36:58,323 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:36:58,323 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): 3276e02ae096deef8165bc8c50c74666/C is initiating minor compaction (all files) 2024-11-27T21:36:58,323 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3276e02ae096deef8165bc8c50c74666/C in TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:58,323 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/76d836b64de84593bb94b1ab631b84bb, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/4fcb4f3258614156bd6d6f31a10422b7, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/1cdae9fe5ac84b0ea9b391f0b141e631] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp, totalSize=35.3 K 2024-11-27T21:36:58,323 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76d836b64de84593bb94b1ab631b84bb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732743414145 2024-11-27T21:36:58,324 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fcb4f3258614156bd6d6f31a10422b7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732743414166 2024-11-27T21:36:58,325 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cdae9fe5ac84b0ea9b391f0b141e631, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1732743416354 2024-11-27T21:36:58,341 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3276e02ae096deef8165bc8c50c74666#C#compaction#509 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:36:58,342 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/dae2ffa20c3b433a983b31c91d8ea7fa is 50, key is test_row_0/C:col10/1732743416354/Put/seqid=0 2024-11-27T21:36:58,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-27T21:36:58,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:58,350 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 3276e02ae096deef8165bc8c50c74666 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-27T21:36:58,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742418_1594 (size=12207) 2024-11-27T21:36:58,389 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/dae2ffa20c3b433a983b31c91d8ea7fa as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/dae2ffa20c3b433a983b31c91d8ea7fa 2024-11-27T21:36:58,394 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3276e02ae096deef8165bc8c50c74666/C of 3276e02ae096deef8165bc8c50c74666 into dae2ffa20c3b433a983b31c91d8ea7fa(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:58,394 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3276e02ae096deef8165bc8c50c74666: 2024-11-27T21:36:58,394 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., storeName=3276e02ae096deef8165bc8c50c74666/C, priority=13, startTime=1732743418276; duration=0sec 2024-11-27T21:36:58,394 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:58,394 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3276e02ae096deef8165bc8c50c74666:C 2024-11-27T21:36:58,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112735b12b536dbd480486b080985611f344_3276e02ae096deef8165bc8c50c74666 is 50, key is test_row_0/A:col10/1732743417001/Put/seqid=0 2024-11-27T21:36:58,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742419_1595 (size=12154) 2024-11-27T21:36:58,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:36:58,431 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112735b12b536dbd480486b080985611f344_3276e02ae096deef8165bc8c50c74666 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112735b12b536dbd480486b080985611f344_3276e02ae096deef8165bc8c50c74666 2024-11-27T21:36:58,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/32207999b4534c9b829729875c6c4431, store: [table=TestAcidGuarantees family=A region=3276e02ae096deef8165bc8c50c74666] 2024-11-27T21:36:58,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/32207999b4534c9b829729875c6c4431 is 175, key is test_row_0/A:col10/1732743417001/Put/seqid=0 2024-11-27T21:36:58,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742420_1596 (size=30955) 2024-11-27T21:36:58,706 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/d294f9c02a3f4db38d8ae9e19af5357b as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/d294f9c02a3f4db38d8ae9e19af5357b 2024-11-27T21:36:58,710 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3276e02ae096deef8165bc8c50c74666/B of 3276e02ae096deef8165bc8c50c74666 into d294f9c02a3f4db38d8ae9e19af5357b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:36:58,710 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3276e02ae096deef8165bc8c50c74666: 2024-11-27T21:36:58,710 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., storeName=3276e02ae096deef8165bc8c50c74666/B, priority=13, startTime=1732743418276; duration=0sec 2024-11-27T21:36:58,710 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:36:58,710 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3276e02ae096deef8165bc8c50c74666:B 2024-11-27T21:36:58,843 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/32207999b4534c9b829729875c6c4431 2024-11-27T21:36:58,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/b9e4e29554734108838ef27d25fede4e is 50, key is test_row_0/B:col10/1732743417001/Put/seqid=0 2024-11-27T21:36:58,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742421_1597 (size=12001) 2024-11-27T21:36:59,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on 3276e02ae096deef8165bc8c50c74666 2024-11-27T21:36:59,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. as already flushing 2024-11-27T21:36:59,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:59,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743479206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:59,211 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,211 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,211 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:59,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:59,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45662 deadline: 1732743479207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:59,211 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,211 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,211 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:59,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:59,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743479207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:59,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:59,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:59,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743479208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:59,212 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:59,212 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,212 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:59,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732743479209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:59,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:59,255 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/b9e4e29554734108838ef27d25fede4e 2024-11-27T21:36:59,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-27T21:36:59,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/c852b328929841908a1cb61d74ad2cd2 is 50, key is test_row_0/C:col10/1732743417001/Put/seqid=0 2024-11-27T21:36:59,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742422_1598 (size=12001) 2024-11-27T21:36:59,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:59,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743479313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:59,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:59,316 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45662 deadline: 1732743479313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:59,316 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,316 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:59,316 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,316 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,316 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:59,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:59,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743479323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:59,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:59,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743479324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:59,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:59,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:59,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:59,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732743479324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:59,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:59,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:59,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743479523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:59,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:59,527 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45662 deadline: 1732743479523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:59,527 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,527 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:59,527 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,527 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,527 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:59,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:59,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743479534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:59,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:59,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:59,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743479534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:59,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:59,537 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732743479534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:59,537 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,537 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:59,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:59,672 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/c852b328929841908a1cb61d74ad2cd2 2024-11-27T21:36:59,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/32207999b4534c9b829729875c6c4431 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/32207999b4534c9b829729875c6c4431 2024-11-27T21:36:59,679 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/32207999b4534c9b829729875c6c4431, entries=150, sequenceid=117, filesize=30.2 K 2024-11-27T21:36:59,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/b9e4e29554734108838ef27d25fede4e as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/b9e4e29554734108838ef27d25fede4e 2024-11-27T21:36:59,683 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/b9e4e29554734108838ef27d25fede4e, entries=150, sequenceid=117, filesize=11.7 K 2024-11-27T21:36:59,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/c852b328929841908a1cb61d74ad2cd2 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/c852b328929841908a1cb61d74ad2cd2 2024-11-27T21:36:59,686 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/c852b328929841908a1cb61d74ad2cd2, entries=150, sequenceid=117, filesize=11.7 K 2024-11-27T21:36:59,687 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 3276e02ae096deef8165bc8c50c74666 in 1337ms, sequenceid=117, compaction requested=false 2024-11-27T21:36:59,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 3276e02ae096deef8165bc8c50c74666: 2024-11-27T21:36:59,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:36:59,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-27T21:36:59,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-27T21:36:59,690 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-27T21:36:59,690 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5720 sec 2024-11-27T21:36:59,691 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 2.5760 sec 2024-11-27T21:36:59,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on 3276e02ae096deef8165bc8c50c74666 2024-11-27T21:36:59,836 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:36:59,836 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3276e02ae096deef8165bc8c50c74666 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-27T21:36:59,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411274507f55f34ae4a489187f2d16d9e81ee_3276e02ae096deef8165bc8c50c74666 is 50, key is test_row_0/A:col10/1732743419205/Put/seqid=0 2024-11-27T21:36:59,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742423_1599 (size=17284) 2024-11-27T21:36:59,848 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:36:59,851 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411274507f55f34ae4a489187f2d16d9e81ee_3276e02ae096deef8165bc8c50c74666 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411274507f55f34ae4a489187f2d16d9e81ee_3276e02ae096deef8165bc8c50c74666 2024-11-27T21:36:59,852 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/081612d39c99470aaac7b66b3210b647, store: [table=TestAcidGuarantees family=A region=3276e02ae096deef8165bc8c50c74666] 2024-11-27T21:36:59,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/081612d39c99470aaac7b66b3210b647 is 175, key is test_row_0/A:col10/1732743419205/Put/seqid=0 2024-11-27T21:36:59,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742424_1600 (size=48389) 2024-11-27T21:36:59,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:59,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743479849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:59,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:59,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:59,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743479853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:59,860 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,860 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,860 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:59,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:59,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45662 deadline: 1732743479857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:59,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:59,861 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743479857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:59,861 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,861 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:59,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:59,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732743479858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:59,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:59,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:59,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:59,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743479963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:59,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:59,966 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,966 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743479963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:59,966 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:59,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:59,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45662 deadline: 1732743479963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:59,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:59,966 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,966 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743479964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:59,966 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:59,966 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,966 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,966 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:59,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:36:59,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:36:59,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732743479973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:36:59,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:36:59,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:00,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:00,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743480173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:00,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:00,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743480173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:00,174 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:00,174 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:00,174 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:00,174 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:00,174 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:00,174 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:00,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:00,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45662 deadline: 1732743480174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:00,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:00,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743480174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:00,177 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:00,177 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:00,177 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:00,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:00,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:00,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:00,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:00,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732743480183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:00,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:00,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:00,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:00,257 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=142, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/081612d39c99470aaac7b66b3210b647 2024-11-27T21:37:00,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/7b890771d3164b2690892a42723917d4 is 50, key is test_row_0/B:col10/1732743419205/Put/seqid=0 2024-11-27T21:37:00,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742425_1601 (size=12151) 2024-11-27T21:37:00,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:00,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743480484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:00,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:00,486 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:00,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743480484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:00,486 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:00,486 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:00,486 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:00,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:00,486 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:00,486 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:00,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45662 deadline: 1732743480484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:00,486 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:00,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:00,486 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:00,486 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:00,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743480484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:00,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:00,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:00,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:00,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:00,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732743480493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:00,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:00,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:00,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:00,680 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/7b890771d3164b2690892a42723917d4 2024-11-27T21:37:00,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/04bb7210db34430ca6a73326e737a84e is 50, key is test_row_0/C:col10/1732743419205/Put/seqid=0 2024-11-27T21:37:00,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742426_1602 (size=12151) 2024-11-27T21:37:00,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:00,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743480993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:00,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:00,995 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:00,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743480993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:00,995 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:00,995 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:00,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:00,996 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:00,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45662 deadline: 1732743480993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:00,996 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:00,996 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:00,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:00,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:00,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:00,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743480993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:00,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:00,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:00,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:00,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:01,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:01,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732743481003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:01,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:01,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:01,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:01,094 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/04bb7210db34430ca6a73326e737a84e 2024-11-27T21:37:01,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/081612d39c99470aaac7b66b3210b647 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/081612d39c99470aaac7b66b3210b647 2024-11-27T21:37:01,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/081612d39c99470aaac7b66b3210b647, entries=250, sequenceid=142, filesize=47.3 K 2024-11-27T21:37:01,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/7b890771d3164b2690892a42723917d4 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/7b890771d3164b2690892a42723917d4 2024-11-27T21:37:01,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/7b890771d3164b2690892a42723917d4, entries=150, sequenceid=142, filesize=11.9 K 2024-11-27T21:37:01,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/04bb7210db34430ca6a73326e737a84e as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/04bb7210db34430ca6a73326e737a84e 2024-11-27T21:37:01,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/04bb7210db34430ca6a73326e737a84e, entries=150, sequenceid=142, filesize=11.9 K 2024-11-27T21:37:01,109 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 3276e02ae096deef8165bc8c50c74666 in 1273ms, sequenceid=142, compaction requested=true 2024-11-27T21:37:01,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3276e02ae096deef8165bc8c50c74666: 2024-11-27T21:37:01,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3276e02ae096deef8165bc8c50c74666:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:37:01,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:37:01,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3276e02ae096deef8165bc8c50c74666:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:37:01,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:37:01,109 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:37:01,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3276e02ae096deef8165bc8c50c74666:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:37:01,109 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:37:01,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:37:01,110 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:37:01,110 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110505 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:37:01,110 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): 3276e02ae096deef8165bc8c50c74666/B is initiating minor compaction (all files) 2024-11-27T21:37:01,110 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): 3276e02ae096deef8165bc8c50c74666/A is initiating minor compaction (all files) 2024-11-27T21:37:01,110 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3276e02ae096deef8165bc8c50c74666/B in TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:01,110 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3276e02ae096deef8165bc8c50c74666/A in TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:01,110 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/d294f9c02a3f4db38d8ae9e19af5357b, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/b9e4e29554734108838ef27d25fede4e, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/7b890771d3164b2690892a42723917d4] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp, totalSize=35.5 K 2024-11-27T21:37:01,110 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/993fe8e5e46a4022aca3a9fc28f6cdf1, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/32207999b4534c9b829729875c6c4431, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/081612d39c99470aaac7b66b3210b647] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp, totalSize=107.9 K 2024-11-27T21:37:01,110 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:01,110 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. files: [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/993fe8e5e46a4022aca3a9fc28f6cdf1, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/32207999b4534c9b829729875c6c4431, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/081612d39c99470aaac7b66b3210b647] 2024-11-27T21:37:01,110 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting d294f9c02a3f4db38d8ae9e19af5357b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1732743416354 2024-11-27T21:37:01,110 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 993fe8e5e46a4022aca3a9fc28f6cdf1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1732743416354 2024-11-27T21:37:01,111 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting b9e4e29554734108838ef27d25fede4e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732743417000 2024-11-27T21:37:01,111 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32207999b4534c9b829729875c6c4431, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732743417000 2024-11-27T21:37:01,111 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b890771d3164b2690892a42723917d4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732743419205 2024-11-27T21:37:01,111 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 081612d39c99470aaac7b66b3210b647, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732743419205 2024-11-27T21:37:01,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,117 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3276e02ae096deef8165bc8c50c74666] 2024-11-27T21:37:01,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,122 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3276e02ae096deef8165bc8c50c74666#B#compaction#516 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:37:01,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,123 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/b782427e56554cc28a144aadf18dc273 is 50, key is test_row_0/B:col10/1732743419205/Put/seqid=0 2024-11-27T21:37:01,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,133 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127943fd189671244f394581d573eeb69ae_3276e02ae096deef8165bc8c50c74666 store=[table=TestAcidGuarantees family=A region=3276e02ae096deef8165bc8c50c74666] 2024-11-27T21:37:01,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,142 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127943fd189671244f394581d573eeb69ae_3276e02ae096deef8165bc8c50c74666, store=[table=TestAcidGuarantees family=A region=3276e02ae096deef8165bc8c50c74666] 2024-11-27T21:37:01,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,142 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127943fd189671244f394581d573eeb69ae_3276e02ae096deef8165bc8c50c74666 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3276e02ae096deef8165bc8c50c74666] 2024-11-27T21:37:01,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742427_1603 (size=12459) 2024-11-27T21:37:01,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,200 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/b782427e56554cc28a144aadf18dc273 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/b782427e56554cc28a144aadf18dc273 2024-11-27T21:37:01,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,209 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3276e02ae096deef8165bc8c50c74666/B of 3276e02ae096deef8165bc8c50c74666 into b782427e56554cc28a144aadf18dc273(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:37:01,210 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3276e02ae096deef8165bc8c50c74666: 2024-11-27T21:37:01,210 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., storeName=3276e02ae096deef8165bc8c50c74666/B, priority=13, startTime=1732743421109; duration=0sec 2024-11-27T21:37:01,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,210 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:37:01,210 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3276e02ae096deef8165bc8c50c74666:B 2024-11-27T21:37:01,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,210 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:37:01,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,211 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:37:01,211 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): 3276e02ae096deef8165bc8c50c74666/C is initiating minor compaction (all files) 2024-11-27T21:37:01,211 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3276e02ae096deef8165bc8c50c74666/C in TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:01,211 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/dae2ffa20c3b433a983b31c91d8ea7fa, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/c852b328929841908a1cb61d74ad2cd2, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/04bb7210db34430ca6a73326e737a84e] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp, totalSize=35.5 K 2024-11-27T21:37:01,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,212 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting dae2ffa20c3b433a983b31c91d8ea7fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1732743416354 2024-11-27T21:37:01,212 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting c852b328929841908a1cb61d74ad2cd2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732743417000 2024-11-27T21:37:01,213 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 04bb7210db34430ca6a73326e737a84e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732743419205 2024-11-27T21:37:01,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742428_1604 (size=4469) 2024-11-27T21:37:01,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,215 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3276e02ae096deef8165bc8c50c74666#A#compaction#517 average throughput is 0.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:37:01,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,216 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/2dcbec61973240c9a5f2b1fbd4120265 is 175, key is test_row_0/A:col10/1732743419205/Put/seqid=0 2024-11-27T21:37:01,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742429_1605 (size=31413) 2024-11-27T21:37:01,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,244 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3276e02ae096deef8165bc8c50c74666#C#compaction#518 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:37:01,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,245 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/19c6b967d0c34a14945306a752ccd49b is 50, key is test_row_0/C:col10/1732743419205/Put/seqid=0 2024-11-27T21:37:01,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742430_1606 (size=12459) 2024-11-27T21:37:01,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,281 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/19c6b967d0c34a14945306a752ccd49b as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/19c6b967d0c34a14945306a752ccd49b 2024-11-27T21:37:01,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,288 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3276e02ae096deef8165bc8c50c74666/C of 3276e02ae096deef8165bc8c50c74666 into 19c6b967d0c34a14945306a752ccd49b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:37:01,288 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3276e02ae096deef8165bc8c50c74666: 2024-11-27T21:37:01,288 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., storeName=3276e02ae096deef8165bc8c50c74666/C, priority=13, startTime=1732743421109; duration=0sec 2024-11-27T21:37:01,288 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:37:01,288 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3276e02ae096deef8165bc8c50c74666:C 2024-11-27T21:37:01,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-27T21:37:01,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,294 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:37:01,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,295 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:37:01,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-27T21:37:01,298 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:37:01,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,300 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:37:01,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,300 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:37:01,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-27T21:37:01,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-27T21:37:01,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-27T21:37:01,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:01,455 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 3276e02ae096deef8165bc8c50c74666 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-27T21:37:01,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127a0ec1d362a4e45afa6575be4e8e2c94c_3276e02ae096deef8165bc8c50c74666 is 50, key is test_row_0/A:col10/1732743419854/Put/seqid=0 2024-11-27T21:37:01,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742431_1607 (size=9814) 2024-11-27T21:37:01,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,490 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127a0ec1d362a4e45afa6575be4e8e2c94c_3276e02ae096deef8165bc8c50c74666 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a0ec1d362a4e45afa6575be4e8e2c94c_3276e02ae096deef8165bc8c50c74666 2024-11-27T21:37:01,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/6b4bb3ae076745e8bafa3ba3c433ff32, store: [table=TestAcidGuarantees family=A region=3276e02ae096deef8165bc8c50c74666] 2024-11-27T21:37:01,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/6b4bb3ae076745e8bafa3ba3c433ff32 is 175, key is test_row_0/A:col10/1732743419854/Put/seqid=0 2024-11-27T21:37:01,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742432_1608 (size=22461) 2024-11-27T21:37:01,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,501 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=158, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/6b4bb3ae076745e8bafa3ba3c433ff32 2024-11-27T21:37:01,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/1d32caa913274a94885fab05bb16d7a2 is 50, key is test_row_0/B:col10/1732743419854/Put/seqid=0 2024-11-27T21:37:01,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742433_1609 (size=9757) 2024-11-27T21:37:01,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,529 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/1d32caa913274a94885fab05bb16d7a2 2024-11-27T21:37:01,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/994fa640b6834ae38faca4d5dabc5ea4 is 50, key is test_row_0/C:col10/1732743419854/Put/seqid=0 2024-11-27T21:37:01,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742434_1610 (size=9757) 2024-11-27T21:37:01,583 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/994fa640b6834ae38faca4d5dabc5ea4 2024-11-27T21:37:01,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/6b4bb3ae076745e8bafa3ba3c433ff32 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/6b4bb3ae076745e8bafa3ba3c433ff32 2024-11-27T21:37:01,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,591 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/6b4bb3ae076745e8bafa3ba3c433ff32, entries=100, sequenceid=158, filesize=21.9 K 2024-11-27T21:37:01,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/1d32caa913274a94885fab05bb16d7a2 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/1d32caa913274a94885fab05bb16d7a2 2024-11-27T21:37:01,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,596 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/1d32caa913274a94885fab05bb16d7a2, entries=100, sequenceid=158, filesize=9.5 K 2024-11-27T21:37:01,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/994fa640b6834ae38faca4d5dabc5ea4 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/994fa640b6834ae38faca4d5dabc5ea4 2024-11-27T21:37:01,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,600 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/994fa640b6834ae38faca4d5dabc5ea4, entries=100, sequenceid=158, filesize=9.5 K 2024-11-27T21:37:01,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,601 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=0 B/0 for 3276e02ae096deef8165bc8c50c74666 in 146ms, sequenceid=158, compaction requested=false 2024-11-27T21:37:01,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 3276e02ae096deef8165bc8c50c74666: 2024-11-27T21:37:01,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:01,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-27T21:37:01,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-27T21:37:01,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,605 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-27T21:37:01,605 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 302 msec 2024-11-27T21:37:01,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,608 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 311 msec 2024-11-27T21:37:01,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-27T21:37:01,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,614 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:37:01,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,614 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:37:01,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-27T21:37:01,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-27T21:37:01,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,617 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:37:01,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,617 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:37:01,617 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:37:01,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,649 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/2dcbec61973240c9a5f2b1fbd4120265 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/2dcbec61973240c9a5f2b1fbd4120265 2024-11-27T21:37:01,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,653 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3276e02ae096deef8165bc8c50c74666/A of 3276e02ae096deef8165bc8c50c74666 into 2dcbec61973240c9a5f2b1fbd4120265(size=30.7 K), total size for store is 52.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:37:01,653 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3276e02ae096deef8165bc8c50c74666: 2024-11-27T21:37:01,653 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., storeName=3276e02ae096deef8165bc8c50c74666/A, priority=13, startTime=1732743421109; duration=0sec 2024-11-27T21:37:01,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,653 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:37:01,653 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3276e02ae096deef8165bc8c50c74666:A 2024-11-27T21:37:01,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-27T21:37:01,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-27T21:37:01,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:01,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for 3276e02ae096deef8165bc8c50c74666: 2024-11-27T21:37:01,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:01,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-27T21:37:01,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-27T21:37:01,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,774 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-27T21:37:01,774 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-11-27T21:37:01,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,776 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 161 msec 2024-11-27T21:37:01,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-27T21:37:01,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,934 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:37:01,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,934 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:37:01,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees 2024-11-27T21:37:01,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-27T21:37:01,937 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:37:01,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,937 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:37:01,938 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:37:01,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:01,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-27T21:37:02,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on 3276e02ae096deef8165bc8c50c74666 2024-11-27T21:37:02,044 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:37:02,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,044 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3276e02ae096deef8165bc8c50c74666 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T21:37:02,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411277a1a426f2fb54c288cb9dbbf2496360a_3276e02ae096deef8165bc8c50c74666 is 50, key is test_row_0/A:col10/1732743422042/Put/seqid=0 2024-11-27T21:37:02,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742436_1612 (size=24758) 2024-11-27T21:37:02,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,084 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:02,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743482079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:02,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:02,089 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411277a1a426f2fb54c288cb9dbbf2496360a_3276e02ae096deef8165bc8c50c74666 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411277a1a426f2fb54c288cb9dbbf2496360a_3276e02ae096deef8165bc8c50c74666 2024-11-27T21:37:02,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743482080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:02,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:02,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743482081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:02,090 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:02,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,090 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:02,090 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:02,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-27T21:37:02,090 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:02,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:02,090 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:02,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:02,091 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:02,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:02,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:02,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:02,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732743482083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:02,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. as already flushing 2024-11-27T21:37:02,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:02,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:37:02,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:37:02,092 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/04af21ca0380432585bdc020d18cf49c, store: [table=TestAcidGuarantees family=A region=3276e02ae096deef8165bc8c50c74666] 2024-11-27T21:37:02,092 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:02,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:02,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:02,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:37:02,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:02,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45662 deadline: 1732743482084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:02,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,095 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:02,095 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:02,095 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:02,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/04af21ca0380432585bdc020d18cf49c is 175, key is test_row_0/A:col10/1732743422042/Put/seqid=0 2024-11-27T21:37:02,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742435_1611 (size=74395) 2024-11-27T21:37:02,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:02,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:02,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:02,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743482193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:02,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743482193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:02,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:02,197 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:02,197 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:02,197 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:02,197 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:02,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743482194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:02,197 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:02,197 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:02,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:02,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:02,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:02,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:02,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732743482203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:02,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:02,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:02,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:02,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:02,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45662 deadline: 1732743482203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:02,207 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:02,207 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:02,207 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:02,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-27T21:37:02,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:02,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. as already flushing 2024-11-27T21:37:02,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:02,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:37:02,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:37:02,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:37:02,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-27T21:37:02,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-27T21:37:02,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:02,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. as already flushing 2024-11-27T21:37:02,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:02,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:37:02,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:37:02,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:37:02,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:02,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743482404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:02,406 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:02,406 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:02,406 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:02,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:02,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743482404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:02,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:02,407 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:02,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743482404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:02,407 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:02,407 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:02,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:02,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:02,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:02,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:02,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732743482413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:02,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:02,417 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:02,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45662 deadline: 1732743482413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:02,417 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:02,417 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:02,417 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:02,417 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:02,417 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:02,499 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=171, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/04af21ca0380432585bdc020d18cf49c 2024-11-27T21:37:02,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/bfef8d0fca284a1ba90037847ba47e09 is 50, key is test_row_0/B:col10/1732743422042/Put/seqid=0 2024-11-27T21:37:02,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742437_1613 (size=12151) 2024-11-27T21:37:02,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-27T21:37:02,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:02,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. as already flushing 2024-11-27T21:37:02,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:02,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:37:02,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:37:02,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:37:02,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-27T21:37:02,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-27T21:37:02,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:02,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. as already flushing 2024-11-27T21:37:02,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:02,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:37:02,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:37:02,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:37:02,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:02,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743482714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:02,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:02,717 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:02,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743482714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:02,717 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:02,718 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:02,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:02,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:02,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743482714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:02,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:02,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:02,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:02,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:02,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:02,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:02,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732743482723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:02,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:02,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:02,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45662 deadline: 1732743482723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:02,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:02,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:02,727 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:02,727 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:02,727 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:02,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-27T21:37:02,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:02,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. as already flushing 2024-11-27T21:37:02,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:02,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:37:02,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:37:02,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:37:02,910 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/bfef8d0fca284a1ba90037847ba47e09 2024-11-27T21:37:02,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/34d2021caf7c46ea96bb547ee1f35930 is 50, key is test_row_0/C:col10/1732743422042/Put/seqid=0 2024-11-27T21:37:02,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742438_1614 (size=12151) 2024-11-27T21:37:03,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-27T21:37:03,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:03,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. as already flushing 2024-11-27T21:37:03,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:03,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:37:03,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:37:03,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:37:03,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-27T21:37:03,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-27T21:37:03,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:03,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. as already flushing 2024-11-27T21:37:03,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:03,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:37:03,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:37:03,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:37:03,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:03,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743483223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:03,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:03,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743483223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:03,225 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:03,225 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:03,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:03,226 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:03,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:03,226 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:03,226 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:03,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743483223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:03,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:03,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:03,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:03,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:03,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732743483234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:03,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:03,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:03,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:03,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:03,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45662 deadline: 1732743483234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:03,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:03,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:03,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:03,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-27T21:37:03,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:03,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. as already flushing 2024-11-27T21:37:03,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:03,315 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:37:03,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:37:03,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:37:03,321 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/34d2021caf7c46ea96bb547ee1f35930 2024-11-27T21:37:03,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/04af21ca0380432585bdc020d18cf49c as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/04af21ca0380432585bdc020d18cf49c 2024-11-27T21:37:03,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/04af21ca0380432585bdc020d18cf49c, entries=400, sequenceid=171, filesize=72.7 K 2024-11-27T21:37:03,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/bfef8d0fca284a1ba90037847ba47e09 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/bfef8d0fca284a1ba90037847ba47e09 2024-11-27T21:37:03,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/bfef8d0fca284a1ba90037847ba47e09, entries=150, sequenceid=171, filesize=11.9 K 2024-11-27T21:37:03,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/34d2021caf7c46ea96bb547ee1f35930 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/34d2021caf7c46ea96bb547ee1f35930 2024-11-27T21:37:03,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/34d2021caf7c46ea96bb547ee1f35930, entries=150, sequenceid=171, filesize=11.9 K 2024-11-27T21:37:03,339 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 3276e02ae096deef8165bc8c50c74666 in 1295ms, sequenceid=171, compaction requested=true 2024-11-27T21:37:03,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3276e02ae096deef8165bc8c50c74666: 2024-11-27T21:37:03,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3276e02ae096deef8165bc8c50c74666:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:37:03,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:37:03,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3276e02ae096deef8165bc8c50c74666:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:37:03,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:37:03,340 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:37:03,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3276e02ae096deef8165bc8c50c74666:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:37:03,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-27T21:37:03,340 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:37:03,341 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 128269 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:37:03,341 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): 3276e02ae096deef8165bc8c50c74666/A is initiating minor compaction (all files) 2024-11-27T21:37:03,341 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3276e02ae096deef8165bc8c50c74666/A in TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:03,341 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/2dcbec61973240c9a5f2b1fbd4120265, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/6b4bb3ae076745e8bafa3ba3c433ff32, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/04af21ca0380432585bdc020d18cf49c] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp, totalSize=125.3 K 2024-11-27T21:37:03,341 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:03,341 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. files: [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/2dcbec61973240c9a5f2b1fbd4120265, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/6b4bb3ae076745e8bafa3ba3c433ff32, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/04af21ca0380432585bdc020d18cf49c] 2024-11-27T21:37:03,341 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:37:03,341 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): 3276e02ae096deef8165bc8c50c74666/B is initiating minor compaction (all files) 2024-11-27T21:37:03,341 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2dcbec61973240c9a5f2b1fbd4120265, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732743419205 2024-11-27T21:37:03,341 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3276e02ae096deef8165bc8c50c74666/B in TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:03,341 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/b782427e56554cc28a144aadf18dc273, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/1d32caa913274a94885fab05bb16d7a2, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/bfef8d0fca284a1ba90037847ba47e09] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp, totalSize=33.6 K 2024-11-27T21:37:03,341 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b4bb3ae076745e8bafa3ba3c433ff32, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732743419852 2024-11-27T21:37:03,342 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting b782427e56554cc28a144aadf18dc273, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732743419205 2024-11-27T21:37:03,342 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04af21ca0380432585bdc020d18cf49c, keycount=400, bloomtype=ROW, size=72.7 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732743422030 2024-11-27T21:37:03,342 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d32caa913274a94885fab05bb16d7a2, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732743419852 2024-11-27T21:37:03,342 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting bfef8d0fca284a1ba90037847ba47e09, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732743422030 2024-11-27T21:37:03,347 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3276e02ae096deef8165bc8c50c74666] 2024-11-27T21:37:03,350 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3276e02ae096deef8165bc8c50c74666#B#compaction#526 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:37:03,350 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/5af5bd2e187741ea9cc0202b25a90df3 is 50, key is test_row_0/B:col10/1732743422042/Put/seqid=0 2024-11-27T21:37:03,353 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411276fe0c74ef7fe4af8becc4899607bab53_3276e02ae096deef8165bc8c50c74666 store=[table=TestAcidGuarantees family=A region=3276e02ae096deef8165bc8c50c74666] 2024-11-27T21:37:03,356 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411276fe0c74ef7fe4af8becc4899607bab53_3276e02ae096deef8165bc8c50c74666, store=[table=TestAcidGuarantees family=A region=3276e02ae096deef8165bc8c50c74666] 2024-11-27T21:37:03,356 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411276fe0c74ef7fe4af8becc4899607bab53_3276e02ae096deef8165bc8c50c74666 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3276e02ae096deef8165bc8c50c74666] 2024-11-27T21:37:03,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742439_1615 (size=12561) 2024-11-27T21:37:03,363 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/5af5bd2e187741ea9cc0202b25a90df3 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/5af5bd2e187741ea9cc0202b25a90df3 2024-11-27T21:37:03,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742440_1616 (size=4469) 2024-11-27T21:37:03,368 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3276e02ae096deef8165bc8c50c74666#A#compaction#525 average throughput is 1.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:37:03,369 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/7addfc8e622145e28b46a4a892ef6699 is 175, key is test_row_0/A:col10/1732743422042/Put/seqid=0 2024-11-27T21:37:03,369 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3276e02ae096deef8165bc8c50c74666/B of 3276e02ae096deef8165bc8c50c74666 into 5af5bd2e187741ea9cc0202b25a90df3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:37:03,369 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3276e02ae096deef8165bc8c50c74666: 2024-11-27T21:37:03,369 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., storeName=3276e02ae096deef8165bc8c50c74666/B, priority=13, startTime=1732743423340; duration=0sec 2024-11-27T21:37:03,369 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:37:03,369 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3276e02ae096deef8165bc8c50c74666:B 2024-11-27T21:37:03,369 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:37:03,370 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:37:03,370 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): 3276e02ae096deef8165bc8c50c74666/C is initiating minor compaction (all files) 2024-11-27T21:37:03,370 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3276e02ae096deef8165bc8c50c74666/C in TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:03,370 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/19c6b967d0c34a14945306a752ccd49b, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/994fa640b6834ae38faca4d5dabc5ea4, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/34d2021caf7c46ea96bb547ee1f35930] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp, totalSize=33.6 K 2024-11-27T21:37:03,370 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 19c6b967d0c34a14945306a752ccd49b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732743419205 2024-11-27T21:37:03,371 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 994fa640b6834ae38faca4d5dabc5ea4, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732743419852 2024-11-27T21:37:03,372 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 34d2021caf7c46ea96bb547ee1f35930, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732743422030 2024-11-27T21:37:03,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742441_1617 (size=31515) 2024-11-27T21:37:03,379 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3276e02ae096deef8165bc8c50c74666#C#compaction#527 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:37:03,380 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/18ac5d59c4644ca4af1e0313851e145c is 50, key is test_row_0/C:col10/1732743422042/Put/seqid=0 2024-11-27T21:37:03,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742442_1618 (size=12561) 2024-11-27T21:37:03,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-27T21:37:03,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:03,468 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2902): Flushing 3276e02ae096deef8165bc8c50c74666 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T21:37:03,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d8ab4f45cc704041aecb48673bd70d77_3276e02ae096deef8165bc8c50c74666 is 50, key is test_row_0/A:col10/1732743422064/Put/seqid=0 2024-11-27T21:37:03,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742443_1619 (size=12304) 2024-11-27T21:37:03,778 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/7addfc8e622145e28b46a4a892ef6699 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/7addfc8e622145e28b46a4a892ef6699 2024-11-27T21:37:03,785 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3276e02ae096deef8165bc8c50c74666/A of 3276e02ae096deef8165bc8c50c74666 into 7addfc8e622145e28b46a4a892ef6699(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:37:03,785 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3276e02ae096deef8165bc8c50c74666: 2024-11-27T21:37:03,785 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., storeName=3276e02ae096deef8165bc8c50c74666/A, priority=13, startTime=1732743423340; duration=0sec 2024-11-27T21:37:03,786 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:37:03,786 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3276e02ae096deef8165bc8c50c74666:A 2024-11-27T21:37:03,789 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/18ac5d59c4644ca4af1e0313851e145c as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/18ac5d59c4644ca4af1e0313851e145c 2024-11-27T21:37:03,793 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3276e02ae096deef8165bc8c50c74666/C of 3276e02ae096deef8165bc8c50c74666 into 18ac5d59c4644ca4af1e0313851e145c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:37:03,793 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3276e02ae096deef8165bc8c50c74666: 2024-11-27T21:37:03,793 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., storeName=3276e02ae096deef8165bc8c50c74666/C, priority=13, startTime=1732743423340; duration=0sec 2024-11-27T21:37:03,793 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:37:03,793 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3276e02ae096deef8165bc8c50c74666:C 2024-11-27T21:37:03,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:03,884 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d8ab4f45cc704041aecb48673bd70d77_3276e02ae096deef8165bc8c50c74666 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d8ab4f45cc704041aecb48673bd70d77_3276e02ae096deef8165bc8c50c74666 2024-11-27T21:37:03,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/702e586c062648309e8b0506b0305b77, store: [table=TestAcidGuarantees family=A region=3276e02ae096deef8165bc8c50c74666] 2024-11-27T21:37:03,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/702e586c062648309e8b0506b0305b77 is 175, key is test_row_0/A:col10/1732743422064/Put/seqid=0 2024-11-27T21:37:03,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742444_1620 (size=31105) 2024-11-27T21:37:03,890 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=197, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/702e586c062648309e8b0506b0305b77 2024-11-27T21:37:03,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/e2d29dee58124da9a65f47d2a4a280c6 is 50, key is test_row_0/B:col10/1732743422064/Put/seqid=0 2024-11-27T21:37:03,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742445_1621 (size=12151) 2024-11-27T21:37:03,901 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/e2d29dee58124da9a65f47d2a4a280c6 2024-11-27T21:37:03,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/4d3d93ccad2c4da091dd14ab6492f278 is 50, key is test_row_0/C:col10/1732743422064/Put/seqid=0 2024-11-27T21:37:03,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742446_1622 (size=12151) 2024-11-27T21:37:04,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-27T21:37:04,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on 3276e02ae096deef8165bc8c50c74666 2024-11-27T21:37:04,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. as already flushing 2024-11-27T21:37:04,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:04,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743484242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:04,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:04,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:04,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:04,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:04,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743484245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:04,247 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:04,247 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:04,247 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:04,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:04,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743484246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:04,249 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:04,250 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:04,250 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:04,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:04,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732743484253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:04,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:04,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45662 deadline: 1732743484254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:04,255 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,39097,1732743267048 for 'test_row_0' in 3276e02ae096deef8165bc8c50c74666 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:37:04,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:04,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:04,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:04,256 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,39097,1732743267048 for 'test_row_0' in 3276e02ae096deef8165bc8c50c74666 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:37:04,256 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:04,256 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:04,256 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:04,315 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/4d3d93ccad2c4da091dd14ab6492f278 2024-11-27T21:37:04,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/702e586c062648309e8b0506b0305b77 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/702e586c062648309e8b0506b0305b77 2024-11-27T21:37:04,322 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/702e586c062648309e8b0506b0305b77, entries=150, sequenceid=197, filesize=30.4 K 2024-11-27T21:37:04,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/e2d29dee58124da9a65f47d2a4a280c6 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/e2d29dee58124da9a65f47d2a4a280c6 2024-11-27T21:37:04,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,326 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/e2d29dee58124da9a65f47d2a4a280c6, entries=150, sequenceid=197, filesize=11.9 K 2024-11-27T21:37:04,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/4d3d93ccad2c4da091dd14ab6492f278 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/4d3d93ccad2c4da091dd14ab6492f278 2024-11-27T21:37:04,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,331 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/4d3d93ccad2c4da091dd14ab6492f278, entries=150, sequenceid=197, filesize=11.9 K 2024-11-27T21:37:04,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,332 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 3276e02ae096deef8165bc8c50c74666 in 864ms, sequenceid=197, compaction requested=false 2024-11-27T21:37:04,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2603): Flush status journal for 3276e02ae096deef8165bc8c50c74666: 2024-11-27T21:37:04,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:04,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=171 2024-11-27T21:37:04,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=171 2024-11-27T21:37:04,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,336 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-27T21:37:04,336 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3960 sec 2024-11-27T21:37:04,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,338 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees in 2.4030 sec 2024-11-27T21:37:04,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on 3276e02ae096deef8165bc8c50c74666 2024-11-27T21:37:04,358 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:37:04,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,358 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3276e02ae096deef8165bc8c50c74666 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-27T21:37:04,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411279cd80def4dbb4a15952d813c956802d5_3276e02ae096deef8165bc8c50c74666 is 50, key is test_row_0/A:col10/1732743424357/Put/seqid=0 2024-11-27T21:37:04,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742447_1623 (size=12304) 2024-11-27T21:37:04,389 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:04,393 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411279cd80def4dbb4a15952d813c956802d5_3276e02ae096deef8165bc8c50c74666 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279cd80def4dbb4a15952d813c956802d5_3276e02ae096deef8165bc8c50c74666 2024-11-27T21:37:04,393 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/0480625fb19b4f0cb3e1b764cd994f58, store: [table=TestAcidGuarantees family=A region=3276e02ae096deef8165bc8c50c74666] 2024-11-27T21:37:04,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/0480625fb19b4f0cb3e1b764cd994f58 is 175, key is test_row_0/A:col10/1732743424357/Put/seqid=0 2024-11-27T21:37:04,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742448_1624 (size=31101) 2024-11-27T21:37:04,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:04,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743484407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:04,412 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:04,412 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:04,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:04,412 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:04,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743484408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:04,413 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:04,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:04,413 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:04,413 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:04,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743484410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:04,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:04,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:04,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:04,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:04,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743484523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:04,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:04,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:04,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:04,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:04,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743484523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:04,527 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:04,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:04,527 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:04,527 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:04,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743484524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:04,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:04,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:04,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:04,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:04,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743484734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:04,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:04,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743484734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:04,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:04,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:04,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:04,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743484734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:04,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:04,737 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:04,737 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:04,737 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:04,738 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:04,738 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:04,738 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:04,802 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/0480625fb19b4f0cb3e1b764cd994f58 2024-11-27T21:37:04,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/00216de008c6429e8e94818992a954f3 is 50, key is test_row_0/B:col10/1732743424357/Put/seqid=0 2024-11-27T21:37:04,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742449_1625 (size=9757) 2024-11-27T21:37:04,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/00216de008c6429e8e94818992a954f3 2024-11-27T21:37:04,821 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/6a3c4eb3c60f49769abfe31ac82852b4 is 50, key is test_row_0/C:col10/1732743424357/Put/seqid=0 2024-11-27T21:37:04,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742450_1626 (size=9757) 2024-11-27T21:37:05,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:05,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743485043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:05,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:05,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:05,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:05,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743485043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:05,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:05,046 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:05,046 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:05,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:05,046 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:05,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743485043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:05,047 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:05,047 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:05,047 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:05,225 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/6a3c4eb3c60f49769abfe31ac82852b4 2024-11-27T21:37:05,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/0480625fb19b4f0cb3e1b764cd994f58 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/0480625fb19b4f0cb3e1b764cd994f58 2024-11-27T21:37:05,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/0480625fb19b4f0cb3e1b764cd994f58, entries=150, sequenceid=211, filesize=30.4 K 2024-11-27T21:37:05,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/00216de008c6429e8e94818992a954f3 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/00216de008c6429e8e94818992a954f3 2024-11-27T21:37:05,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/00216de008c6429e8e94818992a954f3, entries=100, sequenceid=211, filesize=9.5 K 2024-11-27T21:37:05,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/6a3c4eb3c60f49769abfe31ac82852b4 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/6a3c4eb3c60f49769abfe31ac82852b4 2024-11-27T21:37:05,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/6a3c4eb3c60f49769abfe31ac82852b4, entries=100, sequenceid=211, filesize=9.5 K 2024-11-27T21:37:05,241 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 3276e02ae096deef8165bc8c50c74666 in 883ms, sequenceid=211, compaction requested=true 2024-11-27T21:37:05,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3276e02ae096deef8165bc8c50c74666: 2024-11-27T21:37:05,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3276e02ae096deef8165bc8c50c74666:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:37:05,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:37:05,241 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:37:05,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3276e02ae096deef8165bc8c50c74666:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:37:05,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:37:05,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3276e02ae096deef8165bc8c50c74666:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:37:05,241 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:37:05,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:37:05,242 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:37:05,242 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): 3276e02ae096deef8165bc8c50c74666/A is initiating minor compaction (all files) 2024-11-27T21:37:05,242 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3276e02ae096deef8165bc8c50c74666/A in TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:05,242 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/7addfc8e622145e28b46a4a892ef6699, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/702e586c062648309e8b0506b0305b77, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/0480625fb19b4f0cb3e1b764cd994f58] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp, totalSize=91.5 K 2024-11-27T21:37:05,242 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:05,242 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. files: [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/7addfc8e622145e28b46a4a892ef6699, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/702e586c062648309e8b0506b0305b77, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/0480625fb19b4f0cb3e1b764cd994f58] 2024-11-27T21:37:05,243 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34469 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:37:05,243 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7addfc8e622145e28b46a4a892ef6699, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732743422030 2024-11-27T21:37:05,243 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): 3276e02ae096deef8165bc8c50c74666/B is initiating minor compaction (all files) 2024-11-27T21:37:05,243 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3276e02ae096deef8165bc8c50c74666/B in TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:05,243 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/5af5bd2e187741ea9cc0202b25a90df3, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/e2d29dee58124da9a65f47d2a4a280c6, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/00216de008c6429e8e94818992a954f3] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp, totalSize=33.7 K 2024-11-27T21:37:05,243 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 702e586c062648309e8b0506b0305b77, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732743422064 2024-11-27T21:37:05,244 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 5af5bd2e187741ea9cc0202b25a90df3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732743422030 2024-11-27T21:37:05,244 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0480625fb19b4f0cb3e1b764cd994f58, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732743424243 2024-11-27T21:37:05,244 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting e2d29dee58124da9a65f47d2a4a280c6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732743422064 2024-11-27T21:37:05,245 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 00216de008c6429e8e94818992a954f3, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732743424244 2024-11-27T21:37:05,258 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3276e02ae096deef8165bc8c50c74666] 2024-11-27T21:37:05,260 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411275efb153e90a64f088bc3bd06811b3411_3276e02ae096deef8165bc8c50c74666 store=[table=TestAcidGuarantees family=A region=3276e02ae096deef8165bc8c50c74666] 2024-11-27T21:37:05,260 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3276e02ae096deef8165bc8c50c74666#B#compaction#535 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:37:05,261 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/b223c5956c5e44e0943f495d8a6956ff is 50, key is test_row_0/B:col10/1732743424357/Put/seqid=0 2024-11-27T21:37:05,262 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411275efb153e90a64f088bc3bd06811b3411_3276e02ae096deef8165bc8c50c74666, store=[table=TestAcidGuarantees family=A region=3276e02ae096deef8165bc8c50c74666] 2024-11-27T21:37:05,262 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411275efb153e90a64f088bc3bd06811b3411_3276e02ae096deef8165bc8c50c74666 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3276e02ae096deef8165bc8c50c74666] 2024-11-27T21:37:05,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742451_1627 (size=12663) 2024-11-27T21:37:05,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742452_1628 (size=4469) 2024-11-27T21:37:05,289 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3276e02ae096deef8165bc8c50c74666#A#compaction#534 average throughput is 0.79 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:37:05,289 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/fb63c1ec5dae45fd8ea37dc0a57b928c is 175, key is test_row_0/A:col10/1732743424357/Put/seqid=0 2024-11-27T21:37:05,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742453_1629 (size=31724) 2024-11-27T21:37:05,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on 3276e02ae096deef8165bc8c50c74666 2024-11-27T21:37:05,556 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:37:05,556 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3276e02ae096deef8165bc8c50c74666 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T21:37:05,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112778d55682eb5d4294894388c37f31480e_3276e02ae096deef8165bc8c50c74666 is 50, key is test_row_0/A:col10/1732743425555/Put/seqid=0 2024-11-27T21:37:05,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742454_1630 (size=12304) 2024-11-27T21:37:05,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:05,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743485572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:05,576 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:05,576 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:05,576 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:05,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:05,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743485574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:05,579 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:05,579 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:05,579 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:05,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:05,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743485577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:05,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:05,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:05,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:05,675 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/b223c5956c5e44e0943f495d8a6956ff as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/b223c5956c5e44e0943f495d8a6956ff 2024-11-27T21:37:05,680 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3276e02ae096deef8165bc8c50c74666/B of 3276e02ae096deef8165bc8c50c74666 into b223c5956c5e44e0943f495d8a6956ff(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:37:05,680 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3276e02ae096deef8165bc8c50c74666: 2024-11-27T21:37:05,680 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., storeName=3276e02ae096deef8165bc8c50c74666/B, priority=13, startTime=1732743425241; duration=0sec 2024-11-27T21:37:05,680 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:37:05,680 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3276e02ae096deef8165bc8c50c74666:B 2024-11-27T21:37:05,680 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:37:05,681 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34469 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:37:05,681 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): 3276e02ae096deef8165bc8c50c74666/C is initiating minor compaction (all files) 2024-11-27T21:37:05,681 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3276e02ae096deef8165bc8c50c74666/C in TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:05,681 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/18ac5d59c4644ca4af1e0313851e145c, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/4d3d93ccad2c4da091dd14ab6492f278, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/6a3c4eb3c60f49769abfe31ac82852b4] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp, totalSize=33.7 K 2024-11-27T21:37:05,682 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 18ac5d59c4644ca4af1e0313851e145c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732743422030 2024-11-27T21:37:05,682 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d3d93ccad2c4da091dd14ab6492f278, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732743422064 2024-11-27T21:37:05,682 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a3c4eb3c60f49769abfe31ac82852b4, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732743424244 2024-11-27T21:37:05,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:05,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743485684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:05,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:05,688 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:05,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743485684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:05,688 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:05,688 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:05,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:05,689 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:05,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743485684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:05,689 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:05,689 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:05,689 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3276e02ae096deef8165bc8c50c74666#C#compaction#537 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:37:05,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:05,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:05,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:05,689 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/a988146f7d9347cca75b879194800e32 is 50, key is test_row_0/C:col10/1732743424357/Put/seqid=0 2024-11-27T21:37:05,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742455_1631 (size=12663) 2024-11-27T21:37:05,698 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/fb63c1ec5dae45fd8ea37dc0a57b928c as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/fb63c1ec5dae45fd8ea37dc0a57b928c 2024-11-27T21:37:05,701 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3276e02ae096deef8165bc8c50c74666/A of 3276e02ae096deef8165bc8c50c74666 into fb63c1ec5dae45fd8ea37dc0a57b928c(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:37:05,701 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3276e02ae096deef8165bc8c50c74666: 2024-11-27T21:37:05,701 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., storeName=3276e02ae096deef8165bc8c50c74666/A, priority=13, startTime=1732743425241; duration=0sec 2024-11-27T21:37:05,702 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:37:05,702 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3276e02ae096deef8165bc8c50c74666:A 2024-11-27T21:37:05,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:05,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743485893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:05,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:05,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:05,897 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:05,897 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:05,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743485893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:05,897 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:05,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743485893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:05,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:05,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:05,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:05,897 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:05,897 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:05,897 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:05,976 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:05,980 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112778d55682eb5d4294894388c37f31480e_3276e02ae096deef8165bc8c50c74666 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112778d55682eb5d4294894388c37f31480e_3276e02ae096deef8165bc8c50c74666 2024-11-27T21:37:05,981 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/0ea1a28012b64b3c945364b8021625ba, store: [table=TestAcidGuarantees family=A region=3276e02ae096deef8165bc8c50c74666] 2024-11-27T21:37:05,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/0ea1a28012b64b3c945364b8021625ba is 175, key is test_row_0/A:col10/1732743425555/Put/seqid=0 2024-11-27T21:37:05,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742456_1632 (size=31105) 2024-11-27T21:37:05,987 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=236, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/0ea1a28012b64b3c945364b8021625ba 2024-11-27T21:37:05,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/8a9f6136ea3c49b8a0b55d59835c9dd6 is 50, key is test_row_0/B:col10/1732743425555/Put/seqid=0 2024-11-27T21:37:06,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742457_1633 (size=12151) 2024-11-27T21:37:06,005 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/8a9f6136ea3c49b8a0b55d59835c9dd6 2024-11-27T21:37:06,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/360db0300154494bb41830a7f74bd062 is 50, key is test_row_0/C:col10/1732743425555/Put/seqid=0 2024-11-27T21:37:06,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742458_1634 (size=12151) 2024-11-27T21:37:06,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-27T21:37:06,093 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:37:06,094 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:37:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees 2024-11-27T21:37:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-27T21:37:06,096 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:37:06,097 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:37:06,097 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:37:06,099 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/a988146f7d9347cca75b879194800e32 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/a988146f7d9347cca75b879194800e32 2024-11-27T21:37:06,103 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3276e02ae096deef8165bc8c50c74666/C of 3276e02ae096deef8165bc8c50c74666 into a988146f7d9347cca75b879194800e32(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:37:06,103 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3276e02ae096deef8165bc8c50c74666: 2024-11-27T21:37:06,103 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., storeName=3276e02ae096deef8165bc8c50c74666/C, priority=13, startTime=1732743425241; duration=0sec 2024-11-27T21:37:06,103 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:37:06,103 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3276e02ae096deef8165bc8c50c74666:C 2024-11-27T21:37:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-27T21:37:06,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743486203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:06,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:06,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743486203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:06,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:06,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:06,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743486204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:06,205 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:06,205 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:06,205 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:06,205 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:06,206 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:06,206 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:06,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-27T21:37:06,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:06,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. as already flushing 2024-11-27T21:37:06,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:06,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:37:06,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:37:06,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:37:06,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45662 deadline: 1732743486273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:06,275 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,39097,1732743267048 for 'test_row_0' in 3276e02ae096deef8165bc8c50c74666 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:37:06,276 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:06,276 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:06,276 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:06,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732743486284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:06,286 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,39097,1732743267048 for 'test_row_0' in 3276e02ae096deef8165bc8c50c74666 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:37:06,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:06,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:06,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:06,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-27T21:37:06,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:06,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. as already flushing 2024-11-27T21:37:06,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:06,403 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:37:06,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:37:06,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:37:06,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-27T21:37:06,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/360db0300154494bb41830a7f74bd062 2024-11-27T21:37:06,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/0ea1a28012b64b3c945364b8021625ba as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/0ea1a28012b64b3c945364b8021625ba 2024-11-27T21:37:06,424 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/0ea1a28012b64b3c945364b8021625ba, entries=150, sequenceid=236, filesize=30.4 K 2024-11-27T21:37:06,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/8a9f6136ea3c49b8a0b55d59835c9dd6 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/8a9f6136ea3c49b8a0b55d59835c9dd6 2024-11-27T21:37:06,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/8a9f6136ea3c49b8a0b55d59835c9dd6, entries=150, sequenceid=236, filesize=11.9 K 2024-11-27T21:37:06,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/360db0300154494bb41830a7f74bd062 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/360db0300154494bb41830a7f74bd062 2024-11-27T21:37:06,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/360db0300154494bb41830a7f74bd062, entries=150, sequenceid=236, filesize=11.9 K 2024-11-27T21:37:06,435 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 3276e02ae096deef8165bc8c50c74666 in 879ms, sequenceid=236, compaction requested=false 2024-11-27T21:37:06,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3276e02ae096deef8165bc8c50c74666: 2024-11-27T21:37:06,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-27T21:37:06,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:06,556 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2902): Flushing 3276e02ae096deef8165bc8c50c74666 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T21:37:06,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127547b3b3c4c3045af94e5dc96b7143ba6_3276e02ae096deef8165bc8c50c74666 is 50, key is test_row_0/A:col10/1732743425574/Put/seqid=0 2024-11-27T21:37:06,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742459_1635 (size=12304) 2024-11-27T21:37:06,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. as already flushing 2024-11-27T21:37:06,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on 3276e02ae096deef8165bc8c50c74666 2024-11-27T21:37:06,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-27T21:37:06,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:06,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743486779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:06,785 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:06,785 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:06,785 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:06,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:06,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743486785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:06,789 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:06,789 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:06,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:06,789 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:06,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743486785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:06,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:06,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:06,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:06,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:06,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743486893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:06,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:06,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:06,897 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:06,897 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:06,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743486893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:06,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743486893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:06,897 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:06,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:06,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:06,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:06,897 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:06,897 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:06,897 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:06,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:06,969 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127547b3b3c4c3045af94e5dc96b7143ba6_3276e02ae096deef8165bc8c50c74666 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127547b3b3c4c3045af94e5dc96b7143ba6_3276e02ae096deef8165bc8c50c74666 2024-11-27T21:37:06,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/91acd45bbd3f4015ab2898465473a74f, store: [table=TestAcidGuarantees family=A region=3276e02ae096deef8165bc8c50c74666] 2024-11-27T21:37:06,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/91acd45bbd3f4015ab2898465473a74f is 175, key is test_row_0/A:col10/1732743425574/Put/seqid=0 2024-11-27T21:37:06,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742460_1636 (size=31105) 2024-11-27T21:37:07,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:07,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743487104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:07,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:07,108 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:07,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743487104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:07,108 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:07,108 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:07,108 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:07,108 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:07,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:07,108 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:07,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743487104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:07,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:07,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:07,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:07,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-27T21:37:07,378 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=250, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/91acd45bbd3f4015ab2898465473a74f 2024-11-27T21:37:07,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/9d32cad86f8740eeb7fc766600adc4f2 is 50, key is test_row_0/B:col10/1732743425574/Put/seqid=0 2024-11-27T21:37:07,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742461_1637 (size=12151) 2024-11-27T21:37:07,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:07,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743487413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:07,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:07,416 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:07,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743487413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:07,416 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:07,416 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:07,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:07,416 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:07,416 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:07,416 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:07,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743487413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:07,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:07,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:07,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:07,789 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/9d32cad86f8740eeb7fc766600adc4f2 2024-11-27T21:37:07,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/5d810c2473ae457d9c0a19de24fdff43 is 50, key is test_row_0/C:col10/1732743425574/Put/seqid=0 2024-11-27T21:37:07,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742462_1638 (size=12151) 2024-11-27T21:37:07,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:07,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743487923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:07,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:07,927 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:07,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:07,927 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:07,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743487923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:07,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743487923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:07,927 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:07,927 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:07,927 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:07,927 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:07,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:07,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:07,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:08,202 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/5d810c2473ae457d9c0a19de24fdff43 2024-11-27T21:37:08,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/91acd45bbd3f4015ab2898465473a74f as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/91acd45bbd3f4015ab2898465473a74f 2024-11-27T21:37:08,209 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/91acd45bbd3f4015ab2898465473a74f, entries=150, sequenceid=250, filesize=30.4 K 2024-11-27T21:37:08,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/9d32cad86f8740eeb7fc766600adc4f2 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/9d32cad86f8740eeb7fc766600adc4f2 2024-11-27T21:37:08,213 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/9d32cad86f8740eeb7fc766600adc4f2, entries=150, sequenceid=250, filesize=11.9 K 2024-11-27T21:37:08,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/5d810c2473ae457d9c0a19de24fdff43 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/5d810c2473ae457d9c0a19de24fdff43 2024-11-27T21:37:08,216 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/5d810c2473ae457d9c0a19de24fdff43, entries=150, sequenceid=250, filesize=11.9 K 2024-11-27T21:37:08,217 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 3276e02ae096deef8165bc8c50c74666 in 1661ms, sequenceid=250, compaction requested=true 2024-11-27T21:37:08,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2603): Flush status journal for 3276e02ae096deef8165bc8c50c74666: 2024-11-27T21:37:08,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:08,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=173 2024-11-27T21:37:08,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=173 2024-11-27T21:37:08,220 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-27T21:37:08,220 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1210 sec 2024-11-27T21:37:08,221 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees in 2.1260 sec 2024-11-27T21:37:08,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-27T21:37:08,244 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:37:08,244 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:37:08,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=174, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees 2024-11-27T21:37:08,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-27T21:37:08,246 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=174, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:37:08,247 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=174, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:37:08,247 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=175, ppid=174, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:37:08,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-27T21:37:08,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-27T21:37:08,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:08,399 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2902): Flushing 3276e02ae096deef8165bc8c50c74666 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-27T21:37:08,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127ff20ad6343f74945b1a1a820c4630eb6_3276e02ae096deef8165bc8c50c74666 is 50, key is test_row_0/A:col10/1732743426778/Put/seqid=0 2024-11-27T21:37:08,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742463_1639 (size=12454) 2024-11-27T21:37:08,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,412 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127ff20ad6343f74945b1a1a820c4630eb6_3276e02ae096deef8165bc8c50c74666 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127ff20ad6343f74945b1a1a820c4630eb6_3276e02ae096deef8165bc8c50c74666 2024-11-27T21:37:08,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/abd8e6f412644c1f925f973b69070817, store: [table=TestAcidGuarantees family=A region=3276e02ae096deef8165bc8c50c74666] 2024-11-27T21:37:08,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/abd8e6f412644c1f925f973b69070817 is 175, key is test_row_0/A:col10/1732743426778/Put/seqid=0 2024-11-27T21:37:08,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742464_1640 (size=31255) 2024-11-27T21:37:08,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-27T21:37:08,831 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=276, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/abd8e6f412644c1f925f973b69070817 2024-11-27T21:37:08,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/7de3edce756f421f9ec14d2688cb5cfe is 50, key is test_row_0/B:col10/1732743426778/Put/seqid=0 2024-11-27T21:37:08,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742465_1641 (size=12301) 2024-11-27T21:37:08,846 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/7de3edce756f421f9ec14d2688cb5cfe 2024-11-27T21:37:08,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/50578ad0cf70476bb227fbf2fe6d9d92 is 50, key is test_row_0/C:col10/1732743426778/Put/seqid=0 2024-11-27T21:37:08,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742466_1642 (size=12301) 2024-11-27T21:37:08,856 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/50578ad0cf70476bb227fbf2fe6d9d92 2024-11-27T21:37:08,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/abd8e6f412644c1f925f973b69070817 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/abd8e6f412644c1f925f973b69070817 2024-11-27T21:37:08,865 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/abd8e6f412644c1f925f973b69070817, entries=150, sequenceid=276, filesize=30.5 K 2024-11-27T21:37:08,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/7de3edce756f421f9ec14d2688cb5cfe as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/7de3edce756f421f9ec14d2688cb5cfe 2024-11-27T21:37:08,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,869 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/7de3edce756f421f9ec14d2688cb5cfe, entries=150, sequenceid=276, filesize=12.0 K 2024-11-27T21:37:08,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/50578ad0cf70476bb227fbf2fe6d9d92 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/50578ad0cf70476bb227fbf2fe6d9d92 2024-11-27T21:37:08,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-27T21:37:08,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,874 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/50578ad0cf70476bb227fbf2fe6d9d92, entries=150, sequenceid=276, filesize=12.0 K 2024-11-27T21:37:08,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,874 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=0 B/0 for 3276e02ae096deef8165bc8c50c74666 in 475ms, sequenceid=276, compaction requested=true 2024-11-27T21:37:08,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2603): Flush status journal for 3276e02ae096deef8165bc8c50c74666: 2024-11-27T21:37:08,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:08,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=175 2024-11-27T21:37:08,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=175 2024-11-27T21:37:08,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,878 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=175, resume processing ppid=174 2024-11-27T21:37:08,878 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=175, ppid=174, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 629 msec 2024-11-27T21:37:08,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,880 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees in 635 msec 2024-11-27T21:37:08,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:08,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on 3276e02ae096deef8165bc8c50c74666 2024-11-27T21:37:08,991 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:37:08,991 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3276e02ae096deef8165bc8c50c74666 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T21:37:09,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d6bf950d70244ebd839206688a4398f8_3276e02ae096deef8165bc8c50c74666 is 50, key is test_row_0/A:col10/1732743428989/Put/seqid=0 2024-11-27T21:37:09,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742467_1643 (size=12454) 2024-11-27T21:37:09,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:09,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743489046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:09,052 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:09,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:09,052 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:09,053 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:09,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743489046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:09,053 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:09,053 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:09,053 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:09,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:09,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743489052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:09,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:09,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:09,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:09,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:09,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743489164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:09,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:09,167 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:09,167 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:09,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743489164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:09,167 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:09,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:09,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:09,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:09,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:09,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743489164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:09,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:09,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:09,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:09,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:09,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743489373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:09,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:09,377 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743489373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:09,377 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:09,377 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:09,377 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:09,377 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:09,377 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:09,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:09,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743489374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:09,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:09,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:09,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:09,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-27T21:37:09,383 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:37:09,384 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:37:09,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=176, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees 2024-11-27T21:37:09,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-27T21:37:09,385 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=176, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:37:09,386 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=176, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:37:09,386 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=177, ppid=176, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:37:09,415 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:09,418 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d6bf950d70244ebd839206688a4398f8_3276e02ae096deef8165bc8c50c74666 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d6bf950d70244ebd839206688a4398f8_3276e02ae096deef8165bc8c50c74666 2024-11-27T21:37:09,419 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/ee44d6a2e3d546c588298e338aea24b0, store: [table=TestAcidGuarantees family=A region=3276e02ae096deef8165bc8c50c74666] 2024-11-27T21:37:09,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/ee44d6a2e3d546c588298e338aea24b0 is 175, key is test_row_0/A:col10/1732743428989/Put/seqid=0 2024-11-27T21:37:09,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742468_1644 (size=31255) 2024-11-27T21:37:09,423 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=288, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/ee44d6a2e3d546c588298e338aea24b0 2024-11-27T21:37:09,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/7ad658279f0343cc807027e1e9c2d819 is 50, key is test_row_0/B:col10/1732743428989/Put/seqid=0 2024-11-27T21:37:09,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742469_1645 (size=12301) 2024-11-27T21:37:09,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-27T21:37:09,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2024-11-27T21:37:09,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:09,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. as already flushing 2024-11-27T21:37:09,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:09,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] handler.RSProcedureHandler(58): pid=177 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:37:09,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=177 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:37:09,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=177 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:37:09,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:09,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743489683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:09,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:09,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743489683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:09,686 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:09,686 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:09,686 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:09,686 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:09,686 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:09,686 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:09,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:09,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743489683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:09,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:09,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:09,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:09,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2024-11-27T21:37:09,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:09,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. as already flushing 2024-11-27T21:37:09,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:09,691 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] handler.RSProcedureHandler(58): pid=177 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:37:09,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=177 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:37:09,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=177 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:37:09,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-27T21:37:09,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/7ad658279f0343cc807027e1e9c2d819 2024-11-27T21:37:09,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/642385bab380455a94c11d7cefd7e6ca is 50, key is test_row_0/C:col10/1732743428989/Put/seqid=0 2024-11-27T21:37:09,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742470_1646 (size=12301) 2024-11-27T21:37:09,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2024-11-27T21:37:09,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:09,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. as already flushing 2024-11-27T21:37:09,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:09,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] handler.RSProcedureHandler(58): pid=177 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:37:09,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=177 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:37:09,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=177 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:37:09,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2024-11-27T21:37:09,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:09,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. as already flushing 2024-11-27T21:37:09,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:09,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] handler.RSProcedureHandler(58): pid=177 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:37:09,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=177 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:37:09,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=177 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:37:10,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-27T21:37:10,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2024-11-27T21:37:10,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:10,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. as already flushing 2024-11-27T21:37:10,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:10,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] handler.RSProcedureHandler(58): pid=177 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:37:10,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=177 java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:37:10,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4177): Remote procedure failed, pid=177 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:37:10,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:10,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45682 deadline: 1732743490193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:10,195 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:10,195 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:10,195 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:10,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:10,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45672 deadline: 1732743490193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:10,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:10,198 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:10,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:10,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45616 deadline: 1732743490193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:10,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:10,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:10,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:10,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:10,242 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/642385bab380455a94c11d7cefd7e6ca 2024-11-27T21:37:10,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/ee44d6a2e3d546c588298e338aea24b0 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/ee44d6a2e3d546c588298e338aea24b0 2024-11-27T21:37:10,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/ee44d6a2e3d546c588298e338aea24b0, entries=150, sequenceid=288, filesize=30.5 K 2024-11-27T21:37:10,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/7ad658279f0343cc807027e1e9c2d819 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/7ad658279f0343cc807027e1e9c2d819 2024-11-27T21:37:10,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/7ad658279f0343cc807027e1e9c2d819, entries=150, sequenceid=288, filesize=12.0 K 2024-11-27T21:37:10,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/642385bab380455a94c11d7cefd7e6ca as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/642385bab380455a94c11d7cefd7e6ca 2024-11-27T21:37:10,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/642385bab380455a94c11d7cefd7e6ca, entries=150, sequenceid=288, filesize=12.0 K 2024-11-27T21:37:10,257 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 3276e02ae096deef8165bc8c50c74666 in 1265ms, sequenceid=288, compaction requested=true 2024-11-27T21:37:10,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3276e02ae096deef8165bc8c50c74666: 2024-11-27T21:37:10,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3276e02ae096deef8165bc8c50c74666:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:37:10,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:37:10,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3276e02ae096deef8165bc8c50c74666:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:37:10,257 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-27T21:37:10,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:37:10,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3276e02ae096deef8165bc8c50c74666:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:37:10,257 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-27T21:37:10,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:37:10,258 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 156444 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-27T21:37:10,258 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61567 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-27T21:37:10,258 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1541): 3276e02ae096deef8165bc8c50c74666/A is initiating minor compaction (all files) 2024-11-27T21:37:10,258 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): 3276e02ae096deef8165bc8c50c74666/B is initiating minor compaction (all files) 2024-11-27T21:37:10,258 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3276e02ae096deef8165bc8c50c74666/A in TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:10,258 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3276e02ae096deef8165bc8c50c74666/B in TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:10,258 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/fb63c1ec5dae45fd8ea37dc0a57b928c, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/0ea1a28012b64b3c945364b8021625ba, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/91acd45bbd3f4015ab2898465473a74f, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/abd8e6f412644c1f925f973b69070817, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/ee44d6a2e3d546c588298e338aea24b0] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp, totalSize=152.8 K 2024-11-27T21:37:10,258 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/b223c5956c5e44e0943f495d8a6956ff, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/8a9f6136ea3c49b8a0b55d59835c9dd6, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/9d32cad86f8740eeb7fc766600adc4f2, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/7de3edce756f421f9ec14d2688cb5cfe, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/7ad658279f0343cc807027e1e9c2d819] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp, totalSize=60.1 K 2024-11-27T21:37:10,258 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:10,258 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. files: [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/fb63c1ec5dae45fd8ea37dc0a57b928c, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/0ea1a28012b64b3c945364b8021625ba, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/91acd45bbd3f4015ab2898465473a74f, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/abd8e6f412644c1f925f973b69070817, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/ee44d6a2e3d546c588298e338aea24b0] 2024-11-27T21:37:10,259 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting b223c5956c5e44e0943f495d8a6956ff, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732743422078 2024-11-27T21:37:10,259 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb63c1ec5dae45fd8ea37dc0a57b928c, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732743422078 2024-11-27T21:37:10,259 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a9f6136ea3c49b8a0b55d59835c9dd6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732743424405 2024-11-27T21:37:10,259 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ea1a28012b64b3c945364b8021625ba, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732743424405 2024-11-27T21:37:10,259 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d32cad86f8740eeb7fc766600adc4f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732743425556 2024-11-27T21:37:10,259 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91acd45bbd3f4015ab2898465473a74f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732743425556 2024-11-27T21:37:10,259 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 7de3edce756f421f9ec14d2688cb5cfe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732743426778 2024-11-27T21:37:10,260 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ad658279f0343cc807027e1e9c2d819, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732743428989 2024-11-27T21:37:10,260 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting abd8e6f412644c1f925f973b69070817, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732743426778 2024-11-27T21:37:10,260 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee44d6a2e3d546c588298e338aea24b0, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732743428989 2024-11-27T21:37:10,267 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3276e02ae096deef8165bc8c50c74666] 2024-11-27T21:37:10,268 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3276e02ae096deef8165bc8c50c74666#B#compaction#549 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:37:10,269 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/1e2c191ff19949e9b1a1ed876c448907 is 50, key is test_row_0/B:col10/1732743428989/Put/seqid=0 2024-11-27T21:37:10,269 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411271d1e09872a4e4c879c427633e0d85e0c_3276e02ae096deef8165bc8c50c74666 store=[table=TestAcidGuarantees family=A region=3276e02ae096deef8165bc8c50c74666] 2024-11-27T21:37:10,271 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411271d1e09872a4e4c879c427633e0d85e0c_3276e02ae096deef8165bc8c50c74666, store=[table=TestAcidGuarantees family=A region=3276e02ae096deef8165bc8c50c74666] 2024-11-27T21:37:10,271 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411271d1e09872a4e4c879c427633e0d85e0c_3276e02ae096deef8165bc8c50c74666 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3276e02ae096deef8165bc8c50c74666] 2024-11-27T21:37:10,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742471_1647 (size=12983) 2024-11-27T21:37:10,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742472_1648 (size=4469) 2024-11-27T21:37:10,279 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3276e02ae096deef8165bc8c50c74666#A#compaction#550 average throughput is 2.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:37:10,280 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/1478af4ff860438098031a3a23ad0182 is 175, key is test_row_0/A:col10/1732743428989/Put/seqid=0 2024-11-27T21:37:10,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742473_1649 (size=31937) 2024-11-27T21:37:10,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39097 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2024-11-27T21:37:10,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:10,302 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2902): Flushing 3276e02ae096deef8165bc8c50c74666 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-27T21:37:10,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127053028c98d5c41fbb2a9d2397e35c2ee_3276e02ae096deef8165bc8c50c74666 is 50, key is test_row_0/A:col10/1732743429044/Put/seqid=0 2024-11-27T21:37:10,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742474_1650 (size=12454) 2024-11-27T21:37:10,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on 3276e02ae096deef8165bc8c50c74666 2024-11-27T21:37:10,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. as already flushing 2024-11-27T21:37:10,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:10,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45662 deadline: 1732743490339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:10,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:10,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732743490341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:10,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:10,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:10,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:10,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:10,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:10,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:10,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:10,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45662 deadline: 1732743490453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:10,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:10,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732743490453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:10,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:10,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:10,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:10,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:10,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:10,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:10,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-27T21:37:10,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:10,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45662 deadline: 1732743490664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:10,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:37:10,667 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:10,667 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:10,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39097 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732743490664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 2024-11-27T21:37:10,667 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:10,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 , the old value is region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:10,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:37:10,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., hostname=4585c39bed03,39097,1732743267048, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:37:10,679 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/1e2c191ff19949e9b1a1ed876c448907 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/1e2c191ff19949e9b1a1ed876c448907 2024-11-27T21:37:10,683 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 3276e02ae096deef8165bc8c50c74666/B of 3276e02ae096deef8165bc8c50c74666 into 1e2c191ff19949e9b1a1ed876c448907(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:37:10,683 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3276e02ae096deef8165bc8c50c74666: 2024-11-27T21:37:10,683 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., storeName=3276e02ae096deef8165bc8c50c74666/B, priority=11, startTime=1732743430257; duration=0sec 2024-11-27T21:37:10,683 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:37:10,683 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3276e02ae096deef8165bc8c50c74666:B 2024-11-27T21:37:10,683 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-27T21:37:10,684 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61567 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-27T21:37:10,685 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1541): 3276e02ae096deef8165bc8c50c74666/C is initiating minor compaction (all files) 2024-11-27T21:37:10,685 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3276e02ae096deef8165bc8c50c74666/C in TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:10,685 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/a988146f7d9347cca75b879194800e32, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/360db0300154494bb41830a7f74bd062, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/5d810c2473ae457d9c0a19de24fdff43, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/50578ad0cf70476bb227fbf2fe6d9d92, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/642385bab380455a94c11d7cefd7e6ca] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp, totalSize=60.1 K 2024-11-27T21:37:10,685 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting a988146f7d9347cca75b879194800e32, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732743422078 2024-11-27T21:37:10,685 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 360db0300154494bb41830a7f74bd062, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732743424405 2024-11-27T21:37:10,686 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d810c2473ae457d9c0a19de24fdff43, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732743425556 2024-11-27T21:37:10,686 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 50578ad0cf70476bb227fbf2fe6d9d92, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732743426778 2024-11-27T21:37:10,686 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/1478af4ff860438098031a3a23ad0182 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/1478af4ff860438098031a3a23ad0182 2024-11-27T21:37:10,687 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] compactions.Compactor(225): Compacting 642385bab380455a94c11d7cefd7e6ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732743428989 2024-11-27T21:37:10,690 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 3276e02ae096deef8165bc8c50c74666/A of 3276e02ae096deef8165bc8c50c74666 into 1478af4ff860438098031a3a23ad0182(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:37:10,690 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3276e02ae096deef8165bc8c50c74666: 2024-11-27T21:37:10,690 INFO [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., storeName=3276e02ae096deef8165bc8c50c74666/A, priority=11, startTime=1732743430257; duration=0sec 2024-11-27T21:37:10,690 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:37:10,690 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3276e02ae096deef8165bc8c50c74666:A 2024-11-27T21:37:10,694 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3276e02ae096deef8165bc8c50c74666#C#compaction#552 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:37:10,694 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/4ded081ae6054ae2ab91f7004472a5a9 is 50, key is test_row_0/C:col10/1732743428989/Put/seqid=0 2024-11-27T21:37:10,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742475_1651 (size=12983) 2024-11-27T21:37:10,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:10,714 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127053028c98d5c41fbb2a9d2397e35c2ee_3276e02ae096deef8165bc8c50c74666 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127053028c98d5c41fbb2a9d2397e35c2ee_3276e02ae096deef8165bc8c50c74666 2024-11-27T21:37:10,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/cfae4fbf08d147fda8e4ed6e88905ee9, store: [table=TestAcidGuarantees family=A region=3276e02ae096deef8165bc8c50c74666] 2024-11-27T21:37:10,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/cfae4fbf08d147fda8e4ed6e88905ee9 is 175, key is test_row_0/A:col10/1732743429044/Put/seqid=0 2024-11-27T21:37:10,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742476_1652 (size=31255) 2024-11-27T21:37:10,720 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=312, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/cfae4fbf08d147fda8e4ed6e88905ee9 2024-11-27T21:37:10,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/ec53016b2b6e4f24a855afa19204a890 is 50, key is test_row_0/B:col10/1732743429044/Put/seqid=0 2024-11-27T21:37:10,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742477_1653 (size=12301) 2024-11-27T21:37:10,741 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/ec53016b2b6e4f24a855afa19204a890 2024-11-27T21:37:10,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/1bbc791b8dce4d1aa38949bae2bbf676 is 50, key is test_row_0/C:col10/1732743429044/Put/seqid=0 2024-11-27T21:37:10,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742478_1654 (size=12301) 2024-11-27T21:37:10,759 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/1bbc791b8dce4d1aa38949bae2bbf676 2024-11-27T21:37:10,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/cfae4fbf08d147fda8e4ed6e88905ee9 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/cfae4fbf08d147fda8e4ed6e88905ee9 2024-11-27T21:37:10,776 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/cfae4fbf08d147fda8e4ed6e88905ee9, entries=150, sequenceid=312, filesize=30.5 K 2024-11-27T21:37:10,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/ec53016b2b6e4f24a855afa19204a890 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/ec53016b2b6e4f24a855afa19204a890 2024-11-27T21:37:10,781 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/ec53016b2b6e4f24a855afa19204a890, entries=150, sequenceid=312, filesize=12.0 K 2024-11-27T21:37:10,781 INFO [Thread-2523 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2523. 2024-11-27T21:37:10,781 DEBUG [Thread-2523 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:37:10,781 DEBUG [Thread-2523 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:37:10,781 DEBUG [Thread-2523 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:37:10,781 INFO [Thread-2519 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2519. 2024-11-27T21:37:10,781 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:37:10,781 DEBUG [Thread-2519 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:37:10,781 DEBUG [Thread-2519 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:37:10,781 DEBUG [Thread-2519 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:37:10,781 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:37:10,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/1bbc791b8dce4d1aa38949bae2bbf676 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/1bbc791b8dce4d1aa38949bae2bbf676 2024-11-27T21:37:10,783 INFO [Thread-2517 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2517. 2024-11-27T21:37:10,783 DEBUG [Thread-2517 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:37:10,783 DEBUG [Thread-2517 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:37:10,783 DEBUG [Thread-2517 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:37:10,783 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:37:10,785 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/1bbc791b8dce4d1aa38949bae2bbf676, entries=150, sequenceid=312, filesize=12.0 K 2024-11-27T21:37:10,786 INFO [Thread-2515 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2515. 2024-11-27T21:37:10,786 DEBUG [Thread-2515 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:37:10,786 DEBUG [Thread-2515 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:37:10,786 DEBUG [Thread-2515 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:37:10,786 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:37:10,786 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 3276e02ae096deef8165bc8c50c74666 in 484ms, sequenceid=312, compaction requested=false 2024-11-27T21:37:10,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2603): Flush status journal for 3276e02ae096deef8165bc8c50c74666: 2024-11-27T21:37:10,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:10,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=177 2024-11-27T21:37:10,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster(4169): Remote procedure done, pid=177 2024-11-27T21:37:10,788 INFO [Thread-2521 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2521. 2024-11-27T21:37:10,788 DEBUG [Thread-2521 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:37:10,788 DEBUG [Thread-2521 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:37:10,788 DEBUG [Thread-2521 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:37:10,788 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:37:10,789 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=177, resume processing ppid=176 2024-11-27T21:37:10,789 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=177, ppid=176, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4010 sec 2024-11-27T21:37:10,790 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=176, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees in 1.4050 sec 2024-11-27T21:37:10,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39097 {}] regionserver.HRegion(8855): Flush requested on 3276e02ae096deef8165bc8c50c74666 2024-11-27T21:37:10,975 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T21:37:10,975 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3276e02ae096deef8165bc8c50c74666 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-27T21:37:10,975 INFO [Thread-2506 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2506. 2024-11-27T21:37:10,975 DEBUG [Thread-2506 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:37:10,975 DEBUG [Thread-2506 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:37:10,975 DEBUG [Thread-2506 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:37:10,975 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:37:10,976 INFO [Thread-2504 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2504. 2024-11-27T21:37:10,976 DEBUG [Thread-2504 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:37:10,976 DEBUG [Thread-2504 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:37:10,976 DEBUG [Thread-2504 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:37:10,976 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:37:10,980 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112753da4204bd114aeb80f6f6695b54babe_3276e02ae096deef8165bc8c50c74666 is 50, key is test_row_0/A:col10/1732743430974/Put/seqid=0 2024-11-27T21:37:10,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742479_1655 (size=12454) 2024-11-27T21:37:10,984 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:10,986 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112753da4204bd114aeb80f6f6695b54babe_3276e02ae096deef8165bc8c50c74666 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112753da4204bd114aeb80f6f6695b54babe_3276e02ae096deef8165bc8c50c74666 2024-11-27T21:37:10,987 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/42944f3d8b464adbb02d097ed10082a6, store: [table=TestAcidGuarantees family=A region=3276e02ae096deef8165bc8c50c74666] 2024-11-27T21:37:10,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/42944f3d8b464adbb02d097ed10082a6 is 175, key is test_row_0/A:col10/1732743430974/Put/seqid=0 2024-11-27T21:37:10,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742480_1656 (size=31255) 2024-11-27T21:37:11,101 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/4ded081ae6054ae2ab91f7004472a5a9 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/4ded081ae6054ae2ab91f7004472a5a9 2024-11-27T21:37:11,104 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 3276e02ae096deef8165bc8c50c74666/C of 3276e02ae096deef8165bc8c50c74666 into 4ded081ae6054ae2ab91f7004472a5a9(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:37:11,104 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3276e02ae096deef8165bc8c50c74666: 2024-11-27T21:37:11,104 INFO [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., storeName=3276e02ae096deef8165bc8c50c74666/C, priority=11, startTime=1732743430257; duration=0sec 2024-11-27T21:37:11,104 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:37:11,104 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3276e02ae096deef8165bc8c50c74666:C 2024-11-27T21:37:11,205 INFO [Thread-2510 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2510. 2024-11-27T21:37:11,205 INFO [Thread-2512 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2512. 2024-11-27T21:37:11,205 DEBUG [Thread-2510 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:37:11,205 DEBUG [Thread-2512 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:37:11,205 DEBUG [Thread-2510 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:37:11,205 DEBUG [Thread-2512 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:37:11,205 DEBUG [Thread-2512 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:37:11,205 DEBUG [Thread-2510 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:37:11,205 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:37:11,205 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:37:11,215 INFO [Thread-2508 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2508. 2024-11-27T21:37:11,215 DEBUG [Thread-2508 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:37:11,215 DEBUG [Thread-2508 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:37:11,215 DEBUG [Thread-2508 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:37:11,215 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:37:11,391 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=328, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/42944f3d8b464adbb02d097ed10082a6 2024-11-27T21:37:11,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/f571e3d3a6ff4dacb6fa506d44b8177a is 50, key is test_row_0/B:col10/1732743430974/Put/seqid=0 2024-11-27T21:37:11,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742481_1657 (size=12301) 2024-11-27T21:37:11,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-27T21:37:11,533 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:37:11,533 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-27T21:37:11,533 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 36 2024-11-27T21:37:11,533 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 34 2024-11-27T21:37:11,533 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-27T21:37:11,533 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-27T21:37:11,533 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-27T21:37:11,533 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-27T21:37:11,533 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3408 2024-11-27T21:37:11,533 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3468 2024-11-27T21:37:11,533 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3351 2024-11-27T21:37:11,533 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3429 2024-11-27T21:37:11,533 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3469 2024-11-27T21:37:11,533 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-27T21:37:11,533 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T21:37:11,533 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T21:37:11,533 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:37:11,533 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:37:11,534 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:37:11,534 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-27T21:37:11,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=178, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-27T21:37:11,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-27T21:37:11,537 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732743431537"}]},"ts":"1732743431537"} 2024-11-27T21:37:11,538 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-27T21:37:11,538 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-27T21:37:11,538 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=179, ppid=178, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T21:37:11,539 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=180, ppid=179, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3276e02ae096deef8165bc8c50c74666, UNASSIGN}] 2024-11-27T21:37:11,540 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=180, ppid=179, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3276e02ae096deef8165bc8c50c74666, UNASSIGN 2024-11-27T21:37:11,540 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=180 updating hbase:meta row=3276e02ae096deef8165bc8c50c74666, regionState=CLOSING, regionLocation=4585c39bed03,39097,1732743267048 2024-11-27T21:37:11,541 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=180, ppid=179, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3276e02ae096deef8165bc8c50c74666, UNASSIGN because future has completed 2024-11-27T21:37:11,542 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T21:37:11,542 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=181, ppid=180, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048}] 2024-11-27T21:37:11,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-27T21:37:11,694 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] handler.UnassignRegionHandler(122): Close 3276e02ae096deef8165bc8c50c74666 2024-11-27T21:37:11,694 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T21:37:11,694 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegion(1722): Closing 3276e02ae096deef8165bc8c50c74666, disabling compactions & flushes 2024-11-27T21:37:11,694 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:11,799 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/f571e3d3a6ff4dacb6fa506d44b8177a 2024-11-27T21:37:11,804 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/cdd438677a5f4eaba6698e10db56f9a6 is 50, key is test_row_0/C:col10/1732743430974/Put/seqid=0 2024-11-27T21:37:11,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742482_1658 (size=12301) 2024-11-27T21:37:11,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-27T21:37:12,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-27T21:37:12,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/cdd438677a5f4eaba6698e10db56f9a6 2024-11-27T21:37:12,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/42944f3d8b464adbb02d097ed10082a6 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/42944f3d8b464adbb02d097ed10082a6 2024-11-27T21:37:12,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/42944f3d8b464adbb02d097ed10082a6, entries=150, sequenceid=328, filesize=30.5 K 2024-11-27T21:37:12,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/f571e3d3a6ff4dacb6fa506d44b8177a as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/f571e3d3a6ff4dacb6fa506d44b8177a 2024-11-27T21:37:12,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/f571e3d3a6ff4dacb6fa506d44b8177a, entries=150, sequenceid=328, filesize=12.0 K 2024-11-27T21:37:12,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/cdd438677a5f4eaba6698e10db56f9a6 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/cdd438677a5f4eaba6698e10db56f9a6 2024-11-27T21:37:12,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/cdd438677a5f4eaba6698e10db56f9a6, entries=150, sequenceid=328, filesize=12.0 K 2024-11-27T21:37:12,221 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=20.13 KB/20610 for 3276e02ae096deef8165bc8c50c74666 in 1246ms, sequenceid=328, compaction requested=true 2024-11-27T21:37:12,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3276e02ae096deef8165bc8c50c74666: 2024-11-27T21:37:12,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3276e02ae096deef8165bc8c50c74666:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:37:12,221 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:12,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:37:12,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3276e02ae096deef8165bc8c50c74666:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:37:12,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:37:12,221 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:12,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3276e02ae096deef8165bc8c50c74666:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:37:12,221 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. because compaction request was cancelled 2024-11-27T21:37:12,221 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. because compaction request was cancelled 2024-11-27T21:37:12,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:37:12,221 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. after waiting 0 ms 2024-11-27T21:37:12,221 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3276e02ae096deef8165bc8c50c74666:B 2024-11-27T21:37:12,221 DEBUG [RS:0;4585c39bed03:39097-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3276e02ae096deef8165bc8c50c74666:A 2024-11-27T21:37:12,221 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:12,221 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. because compaction request was cancelled 2024-11-27T21:37:12,221 DEBUG [RS:0;4585c39bed03:39097-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3276e02ae096deef8165bc8c50c74666:C 2024-11-27T21:37:12,221 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegion(2902): Flushing 3276e02ae096deef8165bc8c50c74666 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-27T21:37:12,225 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411279938ca933de040118ff12ec13051eeed_3276e02ae096deef8165bc8c50c74666 is 50, key is test_row_0/A:col10/1732743431203/Put/seqid=0 2024-11-27T21:37:12,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742483_1659 (size=9914) 2024-11-27T21:37:12,629 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:37:12,632 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411279938ca933de040118ff12ec13051eeed_3276e02ae096deef8165bc8c50c74666 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279938ca933de040118ff12ec13051eeed_3276e02ae096deef8165bc8c50c74666 2024-11-27T21:37:12,633 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/f9d1a7bc8b8a4ca8a0fa13d2e0ff0880, store: [table=TestAcidGuarantees family=A region=3276e02ae096deef8165bc8c50c74666] 2024-11-27T21:37:12,633 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/f9d1a7bc8b8a4ca8a0fa13d2e0ff0880 is 175, key is test_row_0/A:col10/1732743431203/Put/seqid=0 2024-11-27T21:37:12,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742484_1660 (size=22561) 2024-11-27T21:37:12,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-27T21:37:13,037 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=335, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/f9d1a7bc8b8a4ca8a0fa13d2e0ff0880 2024-11-27T21:37:13,041 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/51b49f0295bf4cd387f5b66537a8e7b4 is 50, key is test_row_0/B:col10/1732743431203/Put/seqid=0 2024-11-27T21:37:13,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742485_1661 (size=9857) 2024-11-27T21:37:13,445 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/51b49f0295bf4cd387f5b66537a8e7b4 2024-11-27T21:37:13,450 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/98b5d7493ebf44c6b07e78b9dcc364cd is 50, key is test_row_0/C:col10/1732743431203/Put/seqid=0 2024-11-27T21:37:13,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742486_1662 (size=9857) 2024-11-27T21:37:13,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-27T21:37:13,853 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/98b5d7493ebf44c6b07e78b9dcc364cd 2024-11-27T21:37:13,856 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/A/f9d1a7bc8b8a4ca8a0fa13d2e0ff0880 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/f9d1a7bc8b8a4ca8a0fa13d2e0ff0880 2024-11-27T21:37:13,859 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/f9d1a7bc8b8a4ca8a0fa13d2e0ff0880, entries=100, sequenceid=335, filesize=22.0 K 2024-11-27T21:37:13,860 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/B/51b49f0295bf4cd387f5b66537a8e7b4 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/51b49f0295bf4cd387f5b66537a8e7b4 2024-11-27T21:37:13,863 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/51b49f0295bf4cd387f5b66537a8e7b4, entries=100, sequenceid=335, filesize=9.6 K 2024-11-27T21:37:13,863 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/.tmp/C/98b5d7493ebf44c6b07e78b9dcc364cd as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/98b5d7493ebf44c6b07e78b9dcc364cd 2024-11-27T21:37:13,866 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/98b5d7493ebf44c6b07e78b9dcc364cd, entries=100, sequenceid=335, filesize=9.6 K 2024-11-27T21:37:13,866 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 3276e02ae096deef8165bc8c50c74666 in 1645ms, sequenceid=335, compaction requested=true 2024-11-27T21:37:13,867 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/a848f715f9ba43bc8c1c545baa1b3090, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/33204ce5f10b4e729cace3f02e00dbc9, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/b0bca8d6fac047e08f9841e426a8db46, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/1655c04c09404ce5923f6ca7d56a2f33, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/978b57872d1c4a048bc742182c2059dc, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/993fe8e5e46a4022aca3a9fc28f6cdf1, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/a1d3edb097024ad9a07896dfe38f99d2, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/32207999b4534c9b829729875c6c4431, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/081612d39c99470aaac7b66b3210b647, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/2dcbec61973240c9a5f2b1fbd4120265, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/6b4bb3ae076745e8bafa3ba3c433ff32, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/04af21ca0380432585bdc020d18cf49c, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/7addfc8e622145e28b46a4a892ef6699, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/702e586c062648309e8b0506b0305b77, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/fb63c1ec5dae45fd8ea37dc0a57b928c, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/0480625fb19b4f0cb3e1b764cd994f58, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/0ea1a28012b64b3c945364b8021625ba, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/91acd45bbd3f4015ab2898465473a74f, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/abd8e6f412644c1f925f973b69070817, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/ee44d6a2e3d546c588298e338aea24b0] to archive 2024-11-27T21:37:13,867 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T21:37:13,868 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/a848f715f9ba43bc8c1c545baa1b3090 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/a848f715f9ba43bc8c1c545baa1b3090 2024-11-27T21:37:13,869 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/33204ce5f10b4e729cace3f02e00dbc9 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/33204ce5f10b4e729cace3f02e00dbc9 2024-11-27T21:37:13,870 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/b0bca8d6fac047e08f9841e426a8db46 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/b0bca8d6fac047e08f9841e426a8db46 2024-11-27T21:37:13,871 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/1655c04c09404ce5923f6ca7d56a2f33 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/1655c04c09404ce5923f6ca7d56a2f33 2024-11-27T21:37:13,871 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/978b57872d1c4a048bc742182c2059dc to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/978b57872d1c4a048bc742182c2059dc 2024-11-27T21:37:13,872 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/993fe8e5e46a4022aca3a9fc28f6cdf1 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/993fe8e5e46a4022aca3a9fc28f6cdf1 2024-11-27T21:37:13,873 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/a1d3edb097024ad9a07896dfe38f99d2 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/a1d3edb097024ad9a07896dfe38f99d2 2024-11-27T21:37:13,874 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/32207999b4534c9b829729875c6c4431 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/32207999b4534c9b829729875c6c4431 2024-11-27T21:37:13,875 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/081612d39c99470aaac7b66b3210b647 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/081612d39c99470aaac7b66b3210b647 2024-11-27T21:37:13,875 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/2dcbec61973240c9a5f2b1fbd4120265 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/2dcbec61973240c9a5f2b1fbd4120265 2024-11-27T21:37:13,876 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/6b4bb3ae076745e8bafa3ba3c433ff32 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/6b4bb3ae076745e8bafa3ba3c433ff32 2024-11-27T21:37:13,877 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/04af21ca0380432585bdc020d18cf49c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/04af21ca0380432585bdc020d18cf49c 2024-11-27T21:37:13,878 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/7addfc8e622145e28b46a4a892ef6699 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/7addfc8e622145e28b46a4a892ef6699 2024-11-27T21:37:13,879 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/702e586c062648309e8b0506b0305b77 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/702e586c062648309e8b0506b0305b77 2024-11-27T21:37:13,879 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/fb63c1ec5dae45fd8ea37dc0a57b928c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/fb63c1ec5dae45fd8ea37dc0a57b928c 2024-11-27T21:37:13,880 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/0480625fb19b4f0cb3e1b764cd994f58 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/0480625fb19b4f0cb3e1b764cd994f58 2024-11-27T21:37:13,881 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/0ea1a28012b64b3c945364b8021625ba to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/0ea1a28012b64b3c945364b8021625ba 2024-11-27T21:37:13,882 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/91acd45bbd3f4015ab2898465473a74f to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/91acd45bbd3f4015ab2898465473a74f 2024-11-27T21:37:13,882 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/abd8e6f412644c1f925f973b69070817 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/abd8e6f412644c1f925f973b69070817 2024-11-27T21:37:13,883 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/ee44d6a2e3d546c588298e338aea24b0 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/ee44d6a2e3d546c588298e338aea24b0 2024-11-27T21:37:13,884 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/2b1051f5e0584c9fa332ab5591a6ba0f, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/a9f94b1a7b254d7193d42e69895a02e2, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/03cd6377235a457e8bc7ae1ec5b0d713, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/08551a63a5ab4e369ab54a5bdd1c93b8, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/eac14c90efa2456ea60ae4db5c4c6006, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/d294f9c02a3f4db38d8ae9e19af5357b, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/e6b4c7a470ee4c05a70c0b5b7b8597e3, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/b9e4e29554734108838ef27d25fede4e, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/b782427e56554cc28a144aadf18dc273, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/7b890771d3164b2690892a42723917d4, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/1d32caa913274a94885fab05bb16d7a2, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/5af5bd2e187741ea9cc0202b25a90df3, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/bfef8d0fca284a1ba90037847ba47e09, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/e2d29dee58124da9a65f47d2a4a280c6, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/b223c5956c5e44e0943f495d8a6956ff, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/00216de008c6429e8e94818992a954f3, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/8a9f6136ea3c49b8a0b55d59835c9dd6, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/9d32cad86f8740eeb7fc766600adc4f2, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/7de3edce756f421f9ec14d2688cb5cfe, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/7ad658279f0343cc807027e1e9c2d819] to archive 2024-11-27T21:37:13,885 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T21:37:13,886 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/2b1051f5e0584c9fa332ab5591a6ba0f to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/2b1051f5e0584c9fa332ab5591a6ba0f 2024-11-27T21:37:13,886 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/a9f94b1a7b254d7193d42e69895a02e2 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/a9f94b1a7b254d7193d42e69895a02e2 2024-11-27T21:37:13,887 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/03cd6377235a457e8bc7ae1ec5b0d713 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/03cd6377235a457e8bc7ae1ec5b0d713 2024-11-27T21:37:13,888 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/08551a63a5ab4e369ab54a5bdd1c93b8 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/08551a63a5ab4e369ab54a5bdd1c93b8 2024-11-27T21:37:13,889 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/eac14c90efa2456ea60ae4db5c4c6006 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/eac14c90efa2456ea60ae4db5c4c6006 2024-11-27T21:37:13,889 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/d294f9c02a3f4db38d8ae9e19af5357b to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/d294f9c02a3f4db38d8ae9e19af5357b 2024-11-27T21:37:13,890 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/e6b4c7a470ee4c05a70c0b5b7b8597e3 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/e6b4c7a470ee4c05a70c0b5b7b8597e3 2024-11-27T21:37:13,891 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/b9e4e29554734108838ef27d25fede4e to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/b9e4e29554734108838ef27d25fede4e 2024-11-27T21:37:13,892 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/b782427e56554cc28a144aadf18dc273 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/b782427e56554cc28a144aadf18dc273 2024-11-27T21:37:13,892 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/7b890771d3164b2690892a42723917d4 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/7b890771d3164b2690892a42723917d4 2024-11-27T21:37:13,893 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/1d32caa913274a94885fab05bb16d7a2 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/1d32caa913274a94885fab05bb16d7a2 2024-11-27T21:37:13,894 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/5af5bd2e187741ea9cc0202b25a90df3 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/5af5bd2e187741ea9cc0202b25a90df3 2024-11-27T21:37:13,894 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/bfef8d0fca284a1ba90037847ba47e09 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/bfef8d0fca284a1ba90037847ba47e09 2024-11-27T21:37:13,895 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/e2d29dee58124da9a65f47d2a4a280c6 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/e2d29dee58124da9a65f47d2a4a280c6 2024-11-27T21:37:13,896 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/b223c5956c5e44e0943f495d8a6956ff to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/b223c5956c5e44e0943f495d8a6956ff 2024-11-27T21:37:13,897 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/00216de008c6429e8e94818992a954f3 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/00216de008c6429e8e94818992a954f3 2024-11-27T21:37:13,898 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/8a9f6136ea3c49b8a0b55d59835c9dd6 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/8a9f6136ea3c49b8a0b55d59835c9dd6 2024-11-27T21:37:13,898 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/9d32cad86f8740eeb7fc766600adc4f2 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/9d32cad86f8740eeb7fc766600adc4f2 2024-11-27T21:37:13,899 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/7de3edce756f421f9ec14d2688cb5cfe to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/7de3edce756f421f9ec14d2688cb5cfe 2024-11-27T21:37:13,900 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/7ad658279f0343cc807027e1e9c2d819 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/7ad658279f0343cc807027e1e9c2d819 2024-11-27T21:37:13,901 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/ce92d251a1024273a919370a09aa4368, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/0602fb9f9ed24086beb1034da1761bcb, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/76d836b64de84593bb94b1ab631b84bb, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/589ecfc0da9044699de21aa620b178c9, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/4fcb4f3258614156bd6d6f31a10422b7, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/dae2ffa20c3b433a983b31c91d8ea7fa, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/1cdae9fe5ac84b0ea9b391f0b141e631, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/c852b328929841908a1cb61d74ad2cd2, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/19c6b967d0c34a14945306a752ccd49b, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/04bb7210db34430ca6a73326e737a84e, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/994fa640b6834ae38faca4d5dabc5ea4, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/18ac5d59c4644ca4af1e0313851e145c, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/34d2021caf7c46ea96bb547ee1f35930, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/4d3d93ccad2c4da091dd14ab6492f278, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/a988146f7d9347cca75b879194800e32, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/6a3c4eb3c60f49769abfe31ac82852b4, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/360db0300154494bb41830a7f74bd062, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/5d810c2473ae457d9c0a19de24fdff43, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/50578ad0cf70476bb227fbf2fe6d9d92, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/642385bab380455a94c11d7cefd7e6ca] to archive 2024-11-27T21:37:13,901 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T21:37:13,902 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/ce92d251a1024273a919370a09aa4368 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/ce92d251a1024273a919370a09aa4368 2024-11-27T21:37:13,903 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/0602fb9f9ed24086beb1034da1761bcb to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/0602fb9f9ed24086beb1034da1761bcb 2024-11-27T21:37:13,904 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/76d836b64de84593bb94b1ab631b84bb to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/76d836b64de84593bb94b1ab631b84bb 2024-11-27T21:37:13,905 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/589ecfc0da9044699de21aa620b178c9 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/589ecfc0da9044699de21aa620b178c9 2024-11-27T21:37:13,905 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/4fcb4f3258614156bd6d6f31a10422b7 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/4fcb4f3258614156bd6d6f31a10422b7 2024-11-27T21:37:13,906 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/dae2ffa20c3b433a983b31c91d8ea7fa to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/dae2ffa20c3b433a983b31c91d8ea7fa 2024-11-27T21:37:13,907 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/1cdae9fe5ac84b0ea9b391f0b141e631 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/1cdae9fe5ac84b0ea9b391f0b141e631 2024-11-27T21:37:13,907 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/c852b328929841908a1cb61d74ad2cd2 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/c852b328929841908a1cb61d74ad2cd2 2024-11-27T21:37:13,908 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/19c6b967d0c34a14945306a752ccd49b to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/19c6b967d0c34a14945306a752ccd49b 2024-11-27T21:37:13,909 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/04bb7210db34430ca6a73326e737a84e to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/04bb7210db34430ca6a73326e737a84e 2024-11-27T21:37:13,910 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/994fa640b6834ae38faca4d5dabc5ea4 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/994fa640b6834ae38faca4d5dabc5ea4 2024-11-27T21:37:13,911 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/18ac5d59c4644ca4af1e0313851e145c to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/18ac5d59c4644ca4af1e0313851e145c 2024-11-27T21:37:13,911 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/34d2021caf7c46ea96bb547ee1f35930 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/34d2021caf7c46ea96bb547ee1f35930 2024-11-27T21:37:13,912 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/4d3d93ccad2c4da091dd14ab6492f278 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/4d3d93ccad2c4da091dd14ab6492f278 2024-11-27T21:37:13,913 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/a988146f7d9347cca75b879194800e32 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/a988146f7d9347cca75b879194800e32 2024-11-27T21:37:13,913 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/6a3c4eb3c60f49769abfe31ac82852b4 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/6a3c4eb3c60f49769abfe31ac82852b4 2024-11-27T21:37:13,914 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/360db0300154494bb41830a7f74bd062 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/360db0300154494bb41830a7f74bd062 2024-11-27T21:37:13,914 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/5d810c2473ae457d9c0a19de24fdff43 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/5d810c2473ae457d9c0a19de24fdff43 2024-11-27T21:37:13,915 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/50578ad0cf70476bb227fbf2fe6d9d92 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/50578ad0cf70476bb227fbf2fe6d9d92 2024-11-27T21:37:13,916 DEBUG [StoreCloser-TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/642385bab380455a94c11d7cefd7e6ca to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/642385bab380455a94c11d7cefd7e6ca 2024-11-27T21:37:13,919 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/recovered.edits/338.seqid, newMaxSeqId=338, maxSeqId=4 2024-11-27T21:37:13,920 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. 2024-11-27T21:37:13,920 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegion(1676): Region close journal for 3276e02ae096deef8165bc8c50c74666: Waiting for close lock at 1732743431694Running coprocessor pre-close hooks at 1732743431694Disabling compacts and flushes for region at 1732743431694Disabling writes for close at 1732743432221 (+527 ms)Obtaining lock to block concurrent updates at 1732743432221Preparing flush snapshotting stores in 3276e02ae096deef8165bc8c50c74666 at 1732743432221Finished memstore snapshotting TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1732743432222 (+1 ms)Flushing stores of TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666. at 1732743432222Flushing 3276e02ae096deef8165bc8c50c74666/A: creating writer at 1732743432222Flushing mob file 3276e02ae096deef8165bc8c50c74666/A: appending metadata at 1732743432225 (+3 ms)Flushing mob file 3276e02ae096deef8165bc8c50c74666/A: closing flushed file at 1732743432225Flushing 3276e02ae096deef8165bc8c50c74666/A: appending metadata at 1732743432633 (+408 ms)Flushing 3276e02ae096deef8165bc8c50c74666/A: closing flushed file at 1732743432633Flushing 3276e02ae096deef8165bc8c50c74666/B: creating writer at 1732743433039 (+406 ms)Flushing 3276e02ae096deef8165bc8c50c74666/B: appending metadata at 1732743433041 (+2 ms)Flushing 3276e02ae096deef8165bc8c50c74666/B: closing flushed file at 1732743433041Flushing 3276e02ae096deef8165bc8c50c74666/C: creating writer at 1732743433448 (+407 ms)Flushing 3276e02ae096deef8165bc8c50c74666/C: appending metadata at 1732743433449 (+1 ms)Flushing 3276e02ae096deef8165bc8c50c74666/C: closing flushed file at 1732743433449Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@414912cc: reopening flushed file at 1732743433856 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7dca5e82: reopening flushed file at 1732743433859 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@403db156: reopening flushed file at 1732743433863 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 3276e02ae096deef8165bc8c50c74666 in 1645ms, sequenceid=335, compaction requested=true at 1732743433866 (+3 ms)Writing region close event to WAL at 1732743433917 (+51 ms)Running coprocessor post-close hooks at 1732743433920 (+3 ms)Closed at 1732743433920 2024-11-27T21:37:13,921 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] handler.UnassignRegionHandler(157): Closed 3276e02ae096deef8165bc8c50c74666 2024-11-27T21:37:13,922 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=180 updating hbase:meta row=3276e02ae096deef8165bc8c50c74666, regionState=CLOSED 2024-11-27T21:37:13,923 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=181, ppid=180, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 because future has completed 2024-11-27T21:37:13,925 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=181, resume processing ppid=180 2024-11-27T21:37:13,925 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=181, ppid=180, state=SUCCESS, hasLock=false; CloseRegionProcedure 3276e02ae096deef8165bc8c50c74666, server=4585c39bed03,39097,1732743267048 in 2.3810 sec 2024-11-27T21:37:13,926 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=180, resume processing ppid=179 2024-11-27T21:37:13,926 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=180, ppid=179, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3276e02ae096deef8165bc8c50c74666, UNASSIGN in 2.3860 sec 2024-11-27T21:37:13,928 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=179, resume processing ppid=178 2024-11-27T21:37:13,928 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=179, ppid=178, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.3890 sec 2024-11-27T21:37:13,929 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732743433929"}]},"ts":"1732743433929"} 2024-11-27T21:37:13,930 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-27T21:37:13,930 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-27T21:37:13,931 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=178, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.3960 sec 2024-11-27T21:37:15,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-27T21:37:15,704 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:37:15,704 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-27T21:37:15,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=182, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:37:15,706 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=182, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:37:15,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=182 2024-11-27T21:37:15,706 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=182, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:37:15,709 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666 2024-11-27T21:37:15,710 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A, FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B, FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C, FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/recovered.edits] 2024-11-27T21:37:15,712 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/1478af4ff860438098031a3a23ad0182 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/1478af4ff860438098031a3a23ad0182 2024-11-27T21:37:15,713 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/42944f3d8b464adbb02d097ed10082a6 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/42944f3d8b464adbb02d097ed10082a6 2024-11-27T21:37:15,714 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/cfae4fbf08d147fda8e4ed6e88905ee9 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/cfae4fbf08d147fda8e4ed6e88905ee9 2024-11-27T21:37:15,715 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/f9d1a7bc8b8a4ca8a0fa13d2e0ff0880 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/A/f9d1a7bc8b8a4ca8a0fa13d2e0ff0880 2024-11-27T21:37:15,716 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/1e2c191ff19949e9b1a1ed876c448907 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/1e2c191ff19949e9b1a1ed876c448907 2024-11-27T21:37:15,717 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/51b49f0295bf4cd387f5b66537a8e7b4 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/51b49f0295bf4cd387f5b66537a8e7b4 2024-11-27T21:37:15,718 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/ec53016b2b6e4f24a855afa19204a890 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/ec53016b2b6e4f24a855afa19204a890 2024-11-27T21:37:15,718 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/f571e3d3a6ff4dacb6fa506d44b8177a to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/B/f571e3d3a6ff4dacb6fa506d44b8177a 2024-11-27T21:37:15,720 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/1bbc791b8dce4d1aa38949bae2bbf676 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/1bbc791b8dce4d1aa38949bae2bbf676 2024-11-27T21:37:15,721 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/4ded081ae6054ae2ab91f7004472a5a9 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/4ded081ae6054ae2ab91f7004472a5a9 2024-11-27T21:37:15,722 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/98b5d7493ebf44c6b07e78b9dcc364cd to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/98b5d7493ebf44c6b07e78b9dcc364cd 2024-11-27T21:37:15,722 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/cdd438677a5f4eaba6698e10db56f9a6 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/C/cdd438677a5f4eaba6698e10db56f9a6 2024-11-27T21:37:15,724 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/recovered.edits/338.seqid to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666/recovered.edits/338.seqid 2024-11-27T21:37:15,724 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/default/TestAcidGuarantees/3276e02ae096deef8165bc8c50c74666 2024-11-27T21:37:15,725 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-27T21:37:15,725 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-27T21:37:15,725 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-27T21:37:15,727 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127053028c98d5c41fbb2a9d2397e35c2ee_3276e02ae096deef8165bc8c50c74666 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127053028c98d5c41fbb2a9d2397e35c2ee_3276e02ae096deef8165bc8c50c74666 2024-11-27T21:37:15,728 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112735b12b536dbd480486b080985611f344_3276e02ae096deef8165bc8c50c74666 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112735b12b536dbd480486b080985611f344_3276e02ae096deef8165bc8c50c74666 2024-11-27T21:37:15,729 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411273932d16fd690413887eef2f257577747_3276e02ae096deef8165bc8c50c74666 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411273932d16fd690413887eef2f257577747_3276e02ae096deef8165bc8c50c74666 2024-11-27T21:37:15,729 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411274507f55f34ae4a489187f2d16d9e81ee_3276e02ae096deef8165bc8c50c74666 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411274507f55f34ae4a489187f2d16d9e81ee_3276e02ae096deef8165bc8c50c74666 2024-11-27T21:37:15,730 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112753da4204bd114aeb80f6f6695b54babe_3276e02ae096deef8165bc8c50c74666 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112753da4204bd114aeb80f6f6695b54babe_3276e02ae096deef8165bc8c50c74666 2024-11-27T21:37:15,731 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127547b3b3c4c3045af94e5dc96b7143ba6_3276e02ae096deef8165bc8c50c74666 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127547b3b3c4c3045af94e5dc96b7143ba6_3276e02ae096deef8165bc8c50c74666 2024-11-27T21:37:15,732 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112778d55682eb5d4294894388c37f31480e_3276e02ae096deef8165bc8c50c74666 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112778d55682eb5d4294894388c37f31480e_3276e02ae096deef8165bc8c50c74666 2024-11-27T21:37:15,732 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411277a1a426f2fb54c288cb9dbbf2496360a_3276e02ae096deef8165bc8c50c74666 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411277a1a426f2fb54c288cb9dbbf2496360a_3276e02ae096deef8165bc8c50c74666 2024-11-27T21:37:15,733 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279938ca933de040118ff12ec13051eeed_3276e02ae096deef8165bc8c50c74666 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279938ca933de040118ff12ec13051eeed_3276e02ae096deef8165bc8c50c74666 2024-11-27T21:37:15,734 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279cd80def4dbb4a15952d813c956802d5_3276e02ae096deef8165bc8c50c74666 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279cd80def4dbb4a15952d813c956802d5_3276e02ae096deef8165bc8c50c74666 2024-11-27T21:37:15,735 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a0ec1d362a4e45afa6575be4e8e2c94c_3276e02ae096deef8165bc8c50c74666 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a0ec1d362a4e45afa6575be4e8e2c94c_3276e02ae096deef8165bc8c50c74666 2024-11-27T21:37:15,735 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127abf1ef7edad44ea2b19f6db293ed26b5_3276e02ae096deef8165bc8c50c74666 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127abf1ef7edad44ea2b19f6db293ed26b5_3276e02ae096deef8165bc8c50c74666 2024-11-27T21:37:15,736 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d64f160ed82b415986f6408abce62033_3276e02ae096deef8165bc8c50c74666 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d64f160ed82b415986f6408abce62033_3276e02ae096deef8165bc8c50c74666 2024-11-27T21:37:15,737 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d6bf950d70244ebd839206688a4398f8_3276e02ae096deef8165bc8c50c74666 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d6bf950d70244ebd839206688a4398f8_3276e02ae096deef8165bc8c50c74666 2024-11-27T21:37:15,738 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d8ab4f45cc704041aecb48673bd70d77_3276e02ae096deef8165bc8c50c74666 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d8ab4f45cc704041aecb48673bd70d77_3276e02ae096deef8165bc8c50c74666 2024-11-27T21:37:15,738 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127e5753fa982234300b8d027bbe5ff89c7_3276e02ae096deef8165bc8c50c74666 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127e5753fa982234300b8d027bbe5ff89c7_3276e02ae096deef8165bc8c50c74666 2024-11-27T21:37:15,739 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127e95e11d22565495c8ad351173e573326_3276e02ae096deef8165bc8c50c74666 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127e95e11d22565495c8ad351173e573326_3276e02ae096deef8165bc8c50c74666 2024-11-27T21:37:15,740 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127ff20ad6343f74945b1a1a820c4630eb6_3276e02ae096deef8165bc8c50c74666 to hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127ff20ad6343f74945b1a1a820c4630eb6_3276e02ae096deef8165bc8c50c74666 2024-11-27T21:37:15,740 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-27T21:37:15,742 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=182, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:37:15,744 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-27T21:37:15,746 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-27T21:37:15,747 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=182, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:37:15,747 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-27T21:37:15,747 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732743435747"}]},"ts":"9223372036854775807"} 2024-11-27T21:37:15,748 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-27T21:37:15,748 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 3276e02ae096deef8165bc8c50c74666, NAME => 'TestAcidGuarantees,,1732743407672.3276e02ae096deef8165bc8c50c74666.', STARTKEY => '', ENDKEY => ''}] 2024-11-27T21:37:15,748 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-27T21:37:15,749 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732743435748"}]},"ts":"9223372036854775807"} 2024-11-27T21:37:15,750 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-27T21:37:15,751 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=182, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:37:15,751 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=182, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 47 msec 2024-11-27T21:37:15,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=182 2024-11-27T21:37:15,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-27T21:37:15,814 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:37:15,823 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=242 (was 240) - Thread LEAK? -, OpenFileDescriptor=453 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=389 (was 360) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7885 (was 8256) 2024-11-27T21:37:15,823 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-27T21:37:15,823 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T21:37:15,823 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T21:37:15,823 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:37:15,823 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:37:15,823 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:37:15,823 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-27T21:37:15,823 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=900566320, stopped=false 2024-11-27T21:37:15,824 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=4585c39bed03,34433,1732743266181 2024-11-27T21:37:15,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39097-0x1015b59e6860001, quorum=127.0.0.1:62402, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T21:37:15,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34433-0x1015b59e6860000, quorum=127.0.0.1:62402, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T21:37:15,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39097-0x1015b59e6860001, quorum=127.0.0.1:62402, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T21:37:15,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34433-0x1015b59e6860000, quorum=127.0.0.1:62402, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T21:37:15,826 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T21:37:15,826 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39097-0x1015b59e6860001, quorum=127.0.0.1:62402, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T21:37:15,827 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34433-0x1015b59e6860000, quorum=127.0.0.1:62402, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T21:37:15,827 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T21:37:15,827 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T21:37:15,827 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:37:15,827 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '4585c39bed03,39097,1732743267048' ***** 2024-11-27T21:37:15,827 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-27T21:37:15,827 INFO [RS:0;4585c39bed03:39097 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-27T21:37:15,828 INFO [RS:0;4585c39bed03:39097 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-27T21:37:15,828 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-27T21:37:15,828 INFO [RS:0;4585c39bed03:39097 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-27T21:37:15,828 INFO [RS:0;4585c39bed03:39097 {}] regionserver.HRegionServer(959): stopping server 4585c39bed03,39097,1732743267048 2024-11-27T21:37:15,828 INFO [RS:0;4585c39bed03:39097 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T21:37:15,828 INFO [RS:0;4585c39bed03:39097 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;4585c39bed03:39097. 2024-11-27T21:37:15,828 DEBUG [RS:0;4585c39bed03:39097 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T21:37:15,828 DEBUG [RS:0;4585c39bed03:39097 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:37:15,828 INFO [RS:0;4585c39bed03:39097 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-27T21:37:15,828 INFO [RS:0;4585c39bed03:39097 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-27T21:37:15,828 INFO [RS:0;4585c39bed03:39097 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-27T21:37:15,828 INFO [RS:0;4585c39bed03:39097 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-27T21:37:15,829 INFO [RS:0;4585c39bed03:39097 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-27T21:37:15,829 DEBUG [RS:0;4585c39bed03:39097 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-27T21:37:15,829 DEBUG [RS:0;4585c39bed03:39097 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T21:37:15,829 DEBUG [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-27T21:37:15,829 INFO [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-27T21:37:15,829 DEBUG [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-27T21:37:15,829 DEBUG [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-27T21:37:15,829 DEBUG [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-27T21:37:15,830 INFO [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-27T21:37:15,850 DEBUG [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/hbase/meta/1588230740/.tmp/info/37a97c3928ee46759fb1507718160692 is 95, key is TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576./info:/1732743296870/DeleteFamily/seqid=0 2024-11-27T21:37:15,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742487_1663 (size=6514) 2024-11-27T21:37:15,890 INFO [regionserver/4585c39bed03:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T21:37:16,030 DEBUG [RS:0;4585c39bed03:39097 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T21:37:16,230 DEBUG [RS:0;4585c39bed03:39097 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T21:37:16,254 INFO [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/hbase/meta/1588230740/.tmp/info/37a97c3928ee46759fb1507718160692 2024-11-27T21:37:16,266 INFO [regionserver/4585c39bed03:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-27T21:37:16,267 INFO [regionserver/4585c39bed03:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-27T21:37:16,272 DEBUG [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/hbase/meta/1588230740/.tmp/ns/b40dad6e8d784d36a2c10ba8fae987b2 is 93, key is TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576./ns:/1732743296832/DeleteFamily/seqid=0 2024-11-27T21:37:16,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742488_1664 (size=5906) 2024-11-27T21:37:16,275 INFO [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/hbase/meta/1588230740/.tmp/ns/b40dad6e8d784d36a2c10ba8fae987b2 2024-11-27T21:37:16,291 DEBUG [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/hbase/meta/1588230740/.tmp/rep_barrier/ff7508e55ecd4b3ba361a3458b52cd07 is 102, key is TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576./rep_barrier:/1732743296832/DeleteFamily/seqid=0 2024-11-27T21:37:16,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742489_1665 (size=6025) 2024-11-27T21:37:16,430 DEBUG [RS:0;4585c39bed03:39097 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T21:37:16,630 DEBUG [RS:0;4585c39bed03:39097 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T21:37:16,695 INFO [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/hbase/meta/1588230740/.tmp/rep_barrier/ff7508e55ecd4b3ba361a3458b52cd07 2024-11-27T21:37:16,698 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-27T21:37:16,712 DEBUG [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/hbase/meta/1588230740/.tmp/table/023249c4d9c94a8db87e6723fce16697 is 96, key is TestAcidGuarantees,,1732743270785.ebbbf218e8d25371b122092896fed576./table:/1732743296832/DeleteFamily/seqid=0 2024-11-27T21:37:16,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742490_1666 (size=5936) 2024-11-27T21:37:16,830 DEBUG [RS:0;4585c39bed03:39097 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T21:37:17,030 DEBUG [RS:0;4585c39bed03:39097 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T21:37:17,116 INFO [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/hbase/meta/1588230740/.tmp/table/023249c4d9c94a8db87e6723fce16697 2024-11-27T21:37:17,119 DEBUG [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/hbase/meta/1588230740/.tmp/info/37a97c3928ee46759fb1507718160692 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/hbase/meta/1588230740/info/37a97c3928ee46759fb1507718160692 2024-11-27T21:37:17,122 INFO [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/hbase/meta/1588230740/info/37a97c3928ee46759fb1507718160692, entries=12, sequenceid=90, filesize=6.4 K 2024-11-27T21:37:17,122 DEBUG [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/hbase/meta/1588230740/.tmp/ns/b40dad6e8d784d36a2c10ba8fae987b2 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/hbase/meta/1588230740/ns/b40dad6e8d784d36a2c10ba8fae987b2 2024-11-27T21:37:17,125 INFO [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/hbase/meta/1588230740/ns/b40dad6e8d784d36a2c10ba8fae987b2, entries=8, sequenceid=90, filesize=5.8 K 2024-11-27T21:37:17,125 DEBUG [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/hbase/meta/1588230740/.tmp/rep_barrier/ff7508e55ecd4b3ba361a3458b52cd07 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/hbase/meta/1588230740/rep_barrier/ff7508e55ecd4b3ba361a3458b52cd07 2024-11-27T21:37:17,128 INFO [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/hbase/meta/1588230740/rep_barrier/ff7508e55ecd4b3ba361a3458b52cd07, entries=6, sequenceid=90, filesize=5.9 K 2024-11-27T21:37:17,128 DEBUG [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/hbase/meta/1588230740/.tmp/table/023249c4d9c94a8db87e6723fce16697 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/hbase/meta/1588230740/table/023249c4d9c94a8db87e6723fce16697 2024-11-27T21:37:17,130 INFO [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/hbase/meta/1588230740/table/023249c4d9c94a8db87e6723fce16697, entries=7, sequenceid=90, filesize=5.8 K 2024-11-27T21:37:17,131 INFO [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1302ms, sequenceid=90, compaction requested=false 2024-11-27T21:37:17,134 DEBUG [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-27T21:37:17,135 DEBUG [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-27T21:37:17,135 INFO [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-27T21:37:17,135 DEBUG [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732743435829Running coprocessor pre-close hooks at 1732743435829Disabling compacts and flushes for region at 1732743435829Disabling writes for close at 1732743435829Obtaining lock to block concurrent updates at 1732743435830 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732743435830Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732743435830Flushing stores of hbase:meta,,1.1588230740 at 1732743435830Flushing 1588230740/info: creating writer at 1732743435830Flushing 1588230740/info: appending metadata at 1732743435850 (+20 ms)Flushing 1588230740/info: closing flushed file at 1732743435850Flushing 1588230740/ns: creating writer at 1732743436258 (+408 ms)Flushing 1588230740/ns: appending metadata at 1732743436272 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1732743436272Flushing 1588230740/rep_barrier: creating writer at 1732743436278 (+6 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732743436291 (+13 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732743436291Flushing 1588230740/table: creating writer at 1732743436698 (+407 ms)Flushing 1588230740/table: appending metadata at 1732743436712 (+14 ms)Flushing 1588230740/table: closing flushed file at 1732743436712Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13e756b0: reopening flushed file at 1732743437118 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6dc2ab27: reopening flushed file at 1732743437122 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@108ac5a2: reopening flushed file at 1732743437125 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e39ec74: reopening flushed file at 1732743437128 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1302ms, sequenceid=90, compaction requested=false at 1732743437131 (+3 ms)Writing region close event to WAL at 1732743437132 (+1 ms)Running coprocessor post-close hooks at 1732743437135 (+3 ms)Closed at 1732743437135 2024-11-27T21:37:17,135 DEBUG [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-27T21:37:17,231 INFO [RS:0;4585c39bed03:39097 {}] regionserver.HRegionServer(976): stopping server 4585c39bed03,39097,1732743267048; all regions closed. 2024-11-27T21:37:17,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741834_1010 (size=25344) 2024-11-27T21:37:17,237 DEBUG [RS:0;4585c39bed03:39097 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/oldWALs 2024-11-27T21:37:17,237 INFO [RS:0;4585c39bed03:39097 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 4585c39bed03%2C39097%2C1732743267048.meta:.meta(num 1732743270288) 2024-11-27T21:37:17,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741832_1008 (size=15257295) 2024-11-27T21:37:17,240 DEBUG [RS:0;4585c39bed03:39097 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/oldWALs 2024-11-27T21:37:17,240 INFO [RS:0;4585c39bed03:39097 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 4585c39bed03%2C39097%2C1732743267048:(num 1732743269353) 2024-11-27T21:37:17,240 DEBUG [RS:0;4585c39bed03:39097 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:37:17,240 INFO [RS:0;4585c39bed03:39097 {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T21:37:17,240 INFO [RS:0;4585c39bed03:39097 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T21:37:17,240 INFO [RS:0;4585c39bed03:39097 {}] hbase.ChoreService(370): Chore service for: regionserver/4585c39bed03:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-27T21:37:17,240 INFO [RS:0;4585c39bed03:39097 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T21:37:17,240 INFO [regionserver/4585c39bed03:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T21:37:17,241 INFO [RS:0;4585c39bed03:39097 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39097 2024-11-27T21:37:17,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39097-0x1015b59e6860001, quorum=127.0.0.1:62402, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/4585c39bed03,39097,1732743267048 2024-11-27T21:37:17,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34433-0x1015b59e6860000, quorum=127.0.0.1:62402, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T21:37:17,244 INFO [RS:0;4585c39bed03:39097 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T21:37:17,246 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [4585c39bed03,39097,1732743267048] 2024-11-27T21:37:17,252 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/4585c39bed03,39097,1732743267048 already deleted, retry=false 2024-11-27T21:37:17,252 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 4585c39bed03,39097,1732743267048 expired; onlineServers=0 2024-11-27T21:37:17,252 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '4585c39bed03,34433,1732743266181' ***** 2024-11-27T21:37:17,252 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-27T21:37:17,252 INFO [M:0;4585c39bed03:34433 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T21:37:17,252 INFO [M:0;4585c39bed03:34433 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T21:37:17,252 DEBUG [M:0;4585c39bed03:34433 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-27T21:37:17,252 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-27T21:37:17,252 DEBUG [M:0;4585c39bed03:34433 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-27T21:37:17,252 DEBUG [master/4585c39bed03:0:becomeActiveMaster-HFileCleaner.large.0-1732743269004 {}] cleaner.HFileCleaner(306): Exit Thread[master/4585c39bed03:0:becomeActiveMaster-HFileCleaner.large.0-1732743269004,5,FailOnTimeoutGroup] 2024-11-27T21:37:17,252 DEBUG [master/4585c39bed03:0:becomeActiveMaster-HFileCleaner.small.0-1732743269005 {}] cleaner.HFileCleaner(306): Exit Thread[master/4585c39bed03:0:becomeActiveMaster-HFileCleaner.small.0-1732743269005,5,FailOnTimeoutGroup] 2024-11-27T21:37:17,253 INFO [M:0;4585c39bed03:34433 {}] hbase.ChoreService(370): Chore service for: master/4585c39bed03:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-27T21:37:17,253 INFO [M:0;4585c39bed03:34433 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T21:37:17,253 DEBUG [M:0;4585c39bed03:34433 {}] master.HMaster(1795): Stopping service threads 2024-11-27T21:37:17,253 INFO [M:0;4585c39bed03:34433 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-27T21:37:17,253 INFO [M:0;4585c39bed03:34433 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T21:37:17,253 ERROR [M:0;4585c39bed03:34433 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-5,5,PEWorkerGroup] Thread[IPC Client (879580466) connection to localhost/127.0.0.1:39545 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:39545,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-27T21:37:17,254 INFO [M:0;4585c39bed03:34433 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T21:37:17,254 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-27T21:37:17,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34433-0x1015b59e6860000, quorum=127.0.0.1:62402, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-27T21:37:17,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34433-0x1015b59e6860000, quorum=127.0.0.1:62402, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T21:37:17,254 DEBUG [M:0;4585c39bed03:34433 {}] zookeeper.ZKUtil(347): master:34433-0x1015b59e6860000, quorum=127.0.0.1:62402, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-27T21:37:17,254 WARN [M:0;4585c39bed03:34433 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-27T21:37:17,255 INFO [M:0;4585c39bed03:34433 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/.lastflushedseqids 2024-11-27T21:37:17,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742491_1667 (size=72) 2024-11-27T21:37:17,264 INFO [M:0;4585c39bed03:34433 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T21:37:17,264 INFO [M:0;4585c39bed03:34433 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-27T21:37:17,264 DEBUG [M:0;4585c39bed03:34433 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-27T21:37:17,264 INFO [M:0;4585c39bed03:34433 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T21:37:17,264 DEBUG [M:0;4585c39bed03:34433 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T21:37:17,264 DEBUG [M:0;4585c39bed03:34433 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-27T21:37:17,264 DEBUG [M:0;4585c39bed03:34433 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T21:37:17,264 INFO [M:0;4585c39bed03:34433 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=785.59 KB heapSize=967.14 KB 2024-11-27T21:37:17,279 DEBUG [M:0;4585c39bed03:34433 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/87d5347b499d431d9007453300698cb9 is 82, key is hbase:meta,,1/info:regioninfo/1732743270371/Put/seqid=0 2024-11-27T21:37:17,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742492_1668 (size=5672) 2024-11-27T21:37:17,347 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39097-0x1015b59e6860001, quorum=127.0.0.1:62402, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T21:37:17,347 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39097-0x1015b59e6860001, quorum=127.0.0.1:62402, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T21:37:17,347 INFO [RS:0;4585c39bed03:39097 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T21:37:17,348 INFO [RS:0;4585c39bed03:39097 {}] regionserver.HRegionServer(1031): Exiting; stopping=4585c39bed03,39097,1732743267048; zookeeper connection closed. 2024-11-27T21:37:17,348 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2846049d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2846049d 2024-11-27T21:37:17,348 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-27T21:37:17,683 INFO [M:0;4585c39bed03:34433 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2241 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/87d5347b499d431d9007453300698cb9 2024-11-27T21:37:17,704 DEBUG [M:0;4585c39bed03:34433 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a3064ebe1df1429db72009078b6bc8f5 is 2276, key is \x00\x00\x00\x00\x00\x00\x00!/proc:d/1732743300046/Put/seqid=0 2024-11-27T21:37:17,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742493_1669 (size=46295) 2024-11-27T21:37:18,107 INFO [M:0;4585c39bed03:34433 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=785.03 KB at sequenceid=2241 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a3064ebe1df1429db72009078b6bc8f5 2024-11-27T21:37:18,111 INFO [M:0;4585c39bed03:34433 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a3064ebe1df1429db72009078b6bc8f5 2024-11-27T21:37:18,125 DEBUG [M:0;4585c39bed03:34433 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6c8dcd87d6754afcbbbcc3037ced5475 is 69, key is 4585c39bed03,39097,1732743267048/rs:state/1732743269094/Put/seqid=0 2024-11-27T21:37:18,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073742494_1670 (size=5156) 2024-11-27T21:37:18,529 INFO [M:0;4585c39bed03:34433 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2241 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6c8dcd87d6754afcbbbcc3037ced5475 2024-11-27T21:37:18,532 DEBUG [M:0;4585c39bed03:34433 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/87d5347b499d431d9007453300698cb9 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/87d5347b499d431d9007453300698cb9 2024-11-27T21:37:18,535 INFO [M:0;4585c39bed03:34433 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/87d5347b499d431d9007453300698cb9, entries=8, sequenceid=2241, filesize=5.5 K 2024-11-27T21:37:18,535 DEBUG [M:0;4585c39bed03:34433 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a3064ebe1df1429db72009078b6bc8f5 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a3064ebe1df1429db72009078b6bc8f5 2024-11-27T21:37:18,538 INFO [M:0;4585c39bed03:34433 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a3064ebe1df1429db72009078b6bc8f5 2024-11-27T21:37:18,538 INFO [M:0;4585c39bed03:34433 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a3064ebe1df1429db72009078b6bc8f5, entries=182, sequenceid=2241, filesize=45.2 K 2024-11-27T21:37:18,538 DEBUG [M:0;4585c39bed03:34433 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6c8dcd87d6754afcbbbcc3037ced5475 as hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/6c8dcd87d6754afcbbbcc3037ced5475 2024-11-27T21:37:18,540 INFO [M:0;4585c39bed03:34433 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39545/user/jenkins/test-data/9d2d68d5-78a7-985a-60e6-b04ca4dd450d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/6c8dcd87d6754afcbbbcc3037ced5475, entries=1, sequenceid=2241, filesize=5.0 K 2024-11-27T21:37:18,541 INFO [M:0;4585c39bed03:34433 {}] regionserver.HRegion(3140): Finished flush of dataSize ~785.59 KB/804443, heapSize ~966.84 KB/990048, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1277ms, sequenceid=2241, compaction requested=false 2024-11-27T21:37:18,542 INFO [M:0;4585c39bed03:34433 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T21:37:18,542 DEBUG [M:0;4585c39bed03:34433 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732743437264Disabling compacts and flushes for region at 1732743437264Disabling writes for close at 1732743437264Obtaining lock to block concurrent updates at 1732743437264Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732743437264Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=804443, getHeapSize=990288, getOffHeapSize=0, getCellsCount=2448 at 1732743437264Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732743437265 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732743437265Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732743437279 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732743437279Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732743437686 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732743437703 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732743437703Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732743438111 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732743438125 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732743438125Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@249f6112: reopening flushed file at 1732743438532 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@220aa6b0: reopening flushed file at 1732743438535 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@426446c6: reopening flushed file at 1732743438538 (+3 ms)Finished flush of dataSize ~785.59 KB/804443, heapSize ~966.84 KB/990048, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1277ms, sequenceid=2241, compaction requested=false at 1732743438541 (+3 ms)Writing region close event to WAL at 1732743438542 (+1 ms)Closed at 1732743438542 2024-11-27T21:37:18,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36377 is added to blk_1073741830_1006 (size=951355) 2024-11-27T21:37:18,545 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T21:37:18,545 INFO [M:0;4585c39bed03:34433 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-27T21:37:18,545 INFO [M:0;4585c39bed03:34433 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34433 2024-11-27T21:37:18,545 INFO [M:0;4585c39bed03:34433 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T21:37:18,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34433-0x1015b59e6860000, quorum=127.0.0.1:62402, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T21:37:18,647 INFO [M:0;4585c39bed03:34433 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T21:37:18,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34433-0x1015b59e6860000, quorum=127.0.0.1:62402, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T21:37:18,652 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@292d50c4{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T21:37:18,654 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@667c6eba{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T21:37:18,654 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T21:37:18,655 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@72782cb0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-27T21:37:18,655 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@54a7b62e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/208c7222-23c4-3618-cb4b-4763d96cc3c1/hadoop.log.dir/,STOPPED} 2024-11-27T21:37:18,658 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-27T21:37:18,658 WARN [BP-1289921252-172.17.0.2-1732743263147 heartbeating to localhost/127.0.0.1:39545 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-27T21:37:18,658 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-27T21:37:18,658 WARN [BP-1289921252-172.17.0.2-1732743263147 heartbeating to localhost/127.0.0.1:39545 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1289921252-172.17.0.2-1732743263147 (Datanode Uuid c9e47ae3-d3c7-423b-ba3f-1c4fb447a714) service to localhost/127.0.0.1:39545 2024-11-27T21:37:18,660 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/208c7222-23c4-3618-cb4b-4763d96cc3c1/cluster_148e8983-f870-82f4-55b3-08390f1cbfef/data/data1/current/BP-1289921252-172.17.0.2-1732743263147 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T21:37:18,661 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/208c7222-23c4-3618-cb4b-4763d96cc3c1/cluster_148e8983-f870-82f4-55b3-08390f1cbfef/data/data2/current/BP-1289921252-172.17.0.2-1732743263147 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T21:37:18,661 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-27T21:37:18,672 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2341a969{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-27T21:37:18,672 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d064751{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T21:37:18,672 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T21:37:18,673 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@130d168e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-27T21:37:18,673 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1a58fd8a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/208c7222-23c4-3618-cb4b-4763d96cc3c1/hadoop.log.dir/,STOPPED} 2024-11-27T21:37:18,695 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-27T21:37:18,834 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down